class RDoc::RipperStateLex

Constants

EXPR_ARG
EXPR_ARG_ANY
EXPR_BEG
EXPR_BEG_ANY
EXPR_CLASS
EXPR_CMDARG
EXPR_DOT
EXPR_END
EXPR_ENDARG
EXPR_ENDFN
EXPR_END_ANY
EXPR_FITEM
EXPR_FNAME
EXPR_LABEL
EXPR_LABELED
EXPR_MID
EXPR_NONE
EXPR_VALUE
RIPPER_HAS_LEX_STATE

TODO: Remove this constants after Ruby 2.4 EOL

Public Class Methods

end?(token) click to toggle source
# File lib/rdoc/parser/ripper_state_lex.rb, line 616
def self.end?(token)
  (token[:state] & EXPR_END)
end
new(code) click to toggle source
# File lib/rdoc/parser/ripper_state_lex.rb, line 593
def initialize(code)
  @buf = []
  @heredoc_queue = []
  @inner_lex = InnerStateLex.new(code)
  @inner_lex_enumerator = Enumerator.new do |y|
    @inner_lex.each do |tk|
      y << tk
    end
  end
end
parse(code) click to toggle source
# File lib/rdoc/parser/ripper_state_lex.rb, line 604
def self.parse(code)
  lex = self.new(code)
  tokens = []
  begin
    while tk = lex.get_squashed_tk
      tokens.push tk
    end
  rescue StopIteration
  end
  tokens
end

Public Instance Methods

get_squashed_tk() click to toggle source
# File lib/rdoc/parser/ripper_state_lex.rb, line 318
def get_squashed_tk
  if @buf.empty?
    tk = @inner_lex_enumerator.next
  else
    tk = @buf.shift
  end
  case tk[:kind]
  when :on_symbeg then
    tk = get_symbol_tk(tk)
  when :on_tstring_beg then
    tk = get_string_tk(tk)
  when :on_backtick then
    if (tk[:state] & (EXPR_FNAME | EXPR_ENDFN)) != 0
      @inner_lex.lex_state = EXPR_ARG unless RIPPER_HAS_LEX_STATE
      tk[:kind] = :on_ident
      tk[:state] = Ripper::Lexer.const_defined?(:State) ? Ripper::Lexer::State.new(EXPR_ARG) : EXPR_ARG
    else
      tk = get_string_tk(tk)
    end
  when :on_regexp_beg then
    tk = get_regexp_tk(tk)
  when :on_embdoc_beg then
    tk = get_embdoc_tk(tk)
  when :on_heredoc_beg then
    @heredoc_queue << retrieve_heredoc_info(tk)
    @inner_lex.lex_state = EXPR_END unless RIPPER_HAS_LEX_STATE
  when :on_nl, :on_ignored_nl, :on_comment, :on_heredoc_end then
    if !@heredoc_queue.empty?
      get_heredoc_tk(*@heredoc_queue.shift)
    elsif tk[:text].nil? # :on_ignored_nl sometimes gives nil
      tk[:text] = ''
    end
  when :on_words_beg then
    tk = get_words_tk(tk)
  when :on_qwords_beg then
    tk = get_words_tk(tk)
  when :on_symbols_beg then
    tk = get_words_tk(tk)
  when :on_qsymbols_beg then
    tk = get_words_tk(tk)
  when :on_op then
    if '&.' == tk[:text]
      tk[:kind] = :on_period
    else
      tk = get_op_tk(tk)
    end
  end
  tk
end