module RDoc::Parser::Tokenizer

Tokenize Ruby code as RDoc::Parser::RipperStateLex style types and token squashing. Token squashing is required by RDoc::TokenStream‘s syntax highlighting.

Constants

COMMENT
DEF_METHOD_NAME
DSTRING
FLOAT
HEREDOC_BEG
HEREDOC_CONTENT
HEREDOC_END
IMAGINARY
INTEGER
KEYWORD
NEWLINE
OP
OP_TOKENS
OTHER: This constants and token type map are for compatibility with RDoc::Parser::RipperStateLex.
RATIONAL
REGEXP
SPACE
STRING
SYMBOL
TOKEN_TYPE_MAP
WORDS

Public Class Methods

partial_tokenize (whole_code, node, prism_tokens, prism_comments, start_offset = nil, end_offset = nil)

Source

# File lib/rdoc/parser/tokenizer.rb, line 63
def partial_tokenize(whole_code, node, prism_tokens, prism_comments, start_offset = nil, end_offset = nil)
  start_offset ||= node.location.start_offset
  end_offset ||= node.location.end_offset
  visitor = SquashTokenVisitor.new
  node.accept(visitor)
  squashed_tokens = visitor.tokens
  comment_tokens = comment_tokens(slice_by_location(prism_comments, start_offset, end_offset))
  normal_tokens = normal_tokens(slice_by_location(prism_tokens, start_offset, end_offset))
  prior_tokens = (squashed_tokens + comment_tokens).sort_by {|_, start_offset, _| start_offset }
  unify_tokens(whole_code, prior_tokens, normal_tokens, start_offset, end_offset)
end

tokenize (code)

Source

# File lib/rdoc/parser/tokenizer.rb, line 56
def tokenize(code)
  result = Prism.parse_lex(code)
  program_node, unordered_tokens = result.value
  prism_tokens = unordered_tokens.map(&:first).sort_by! { |token| token.location.start_offset }
  partial_tokenize(code, program_node, prism_tokens, result.comments, 0, code.bytesize)
end