diff --git a/lib/rdoc.rb b/lib/rdoc.rb
index 8482e910f5..3856468421 100644
--- a/lib/rdoc.rb
+++ b/lib/rdoc.rb
@@ -65,7 +65,7 @@ module RDoc
##
# RDoc version you are using
- VERSION = '6.0.0.beta1'
+ VERSION = '6.0.0.beta2'
##
# Method visibilities
@@ -148,7 +148,7 @@ module RDoc
autoload :KNOWN_CLASSES, 'rdoc/known_classes'
- autoload :RubyLex, 'rdoc/ruby_lex'
+ autoload :RipperStateLex, 'rdoc/parser/ripper_state_lex'
autoload :RubyToken, 'rdoc/ruby_token'
autoload :TokenStream, 'rdoc/token_stream'
diff --git a/lib/rdoc/constant.rb b/lib/rdoc/constant.rb
index 4fd5c5f10f..7e1e10da72 100644
--- a/lib/rdoc/constant.rb
+++ b/lib/rdoc/constant.rb
@@ -36,7 +36,7 @@ class RDoc::Constant < RDoc::CodeObject
@value = value
@is_alias_for = nil
- @visibility = nil
+ @visibility = :public
self.comment = comment
end
@@ -136,7 +136,7 @@ class RDoc::Constant < RDoc::CodeObject
initialize array[1], nil, array[5]
@full_name = array[2]
- @visibility = array[3]
+ @visibility = array[3] || :public
@is_alias_for = array[4]
# 5 handled above
# 6 handled below
diff --git a/lib/rdoc/context.rb b/lib/rdoc/context.rb
index 7fd6bb0af9..3294b2561c 100644
--- a/lib/rdoc/context.rb
+++ b/lib/rdoc/context.rb
@@ -1079,6 +1079,7 @@ class RDoc::Context < RDoc::CodeObject
return if [:private, :nodoc].include? min_visibility
remove_invisible_in @method_list, min_visibility
remove_invisible_in @attributes, min_visibility
+ remove_invisible_in @constants, min_visibility
end
##
@@ -1165,6 +1166,17 @@ class RDoc::Context < RDoc::CodeObject
end
end
+ ##
+ # Given an array +names+ of constants, set the visibility of each constant to
+ # +visibility+
+
+ def set_constant_visibility_for(names, visibility)
+ names.each do |name|
+ constant = @constants_hash[name] or next
+ constant.visibility = visibility
+ end
+ end
+
##
# Sorts sections alphabetically (default) or in TomDoc fashion (none,
# Public, Internal, Deprecated)
diff --git a/lib/rdoc/markdown.rb b/lib/rdoc/markdown.rb
index b10f5193da..9b433b3ee6 100644
--- a/lib/rdoc/markdown.rb
+++ b/lib/rdoc/markdown.rb
@@ -15280,7 +15280,7 @@ class RDoc::Markdown
self.pos = _save
break
end
- @result = begin;
+ @result = begin;
ref = [:inline, @note_order.length]
@footnotes[ref] = paragraph a
diff --git a/lib/rdoc/markup/to_html.rb b/lib/rdoc/markup/to_html.rb
index aa5de7bf66..98ed7926fd 100644
--- a/lib/rdoc/markup/to_html.rb
+++ b/lib/rdoc/markup/to_html.rb
@@ -200,10 +200,12 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
content = if verbatim.ruby? or parseable? text then
begin
- tokens = RDoc::RubyLex.tokenize text, @options
+ tokens = RDoc::RipperStateLex.parse text
klass = ' class="ruby"'
- RDoc::TokenStream.to_html tokens
+ result = RDoc::TokenStream.to_html tokens
+ result = result + "\n" unless "\n" == result[-1]
+ result
rescue RDoc::RubyLex::Error
CGI.escapeHTML text
end
@@ -212,7 +214,7 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
end
if @options.pipe then
- @res << "\n#{CGI.escapeHTML text}
\n"
+ @res << "\n#{CGI.escapeHTML text}\n
\n"
else
@res << "\n#{content}\n"
end
diff --git a/lib/rdoc/parser/c.rb b/lib/rdoc/parser/c.rb
index 0e72df2ecd..5c940ab28e 100644
--- a/lib/rdoc/parser/c.rb
+++ b/lib/rdoc/parser/c.rb
@@ -666,8 +666,7 @@ class RDoc::Parser::C < RDoc::Parser
#meth_obj.params = params
meth_obj.start_collecting_tokens
- tk = RDoc::RubyToken::Token.new nil, 1, 1
- tk.set_text body
+ tk = { :line_no => 1, :char_no => 1, :text => body }
meth_obj.add_token tk
meth_obj.comment = comment
meth_obj.line = file_content[0, offset].count("\n") + 1
@@ -684,8 +683,7 @@ class RDoc::Parser::C < RDoc::Parser
find_modifiers comment, meth_obj
meth_obj.start_collecting_tokens
- tk = RDoc::RubyToken::Token.new nil, 1, 1
- tk.set_text body
+ tk = { :line_no => 1, :char_no => 1, :text => body }
meth_obj.add_token tk
meth_obj.comment = comment
meth_obj.line = file_content[0, offset].count("\n") + 1
diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb
new file mode 100644
index 0000000000..c9a0f5a21e
--- /dev/null
+++ b/lib/rdoc/parser/ripper_state_lex.rb
@@ -0,0 +1,587 @@
+require 'ripper'
+
+class RDoc::RipperStateLex
+ EXPR_NONE = 0
+ EXPR_BEG = 1
+ EXPR_END = 2
+ EXPR_ENDARG = 4
+ EXPR_ENDFN = 8
+ EXPR_ARG = 16
+ EXPR_CMDARG = 32
+ EXPR_MID = 64
+ EXPR_FNAME = 128
+ EXPR_DOT = 256
+ EXPR_CLASS = 512
+ EXPR_LABEL = 1024
+ EXPR_LABELED = 2048
+ EXPR_FITEM = 4096
+ EXPR_VALUE = EXPR_BEG
+ EXPR_BEG_ANY = (EXPR_BEG | EXPR_MID | EXPR_CLASS)
+ EXPR_ARG_ANY = (EXPR_ARG | EXPR_CMDARG)
+ EXPR_END_ANY = (EXPR_END | EXPR_ENDARG | EXPR_ENDFN)
+
+ class InnerStateLex < Ripper::Filter
+ attr_accessor :lex_state
+
+ def initialize(code)
+ @lex_state = EXPR_BEG
+ @in_fname = false
+ @continue = false
+ reset
+ super(code)
+ end
+
+ def reset
+ @command_start = false
+ @cmd_state = @command_start
+ end
+
+ def on_nl(tok, data)
+ case @lex_state
+ when EXPR_FNAME, EXPR_DOT
+ @continue = true
+ else
+ @continue = false
+ @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
+ end
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_ignored_nl(tok, data)
+ case @lex_state
+ when EXPR_FNAME, EXPR_DOT
+ @continue = true
+ else
+ @continue = false
+ @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
+ end
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_op(tok, data)
+ case tok
+ when '&', '|', '!', '!=', '!~'
+ case @lex_state
+ when EXPR_FNAME, EXPR_DOT
+ @lex_state = EXPR_ARG
+ else
+ @lex_state = EXPR_BEG
+ end
+ when '<<'
+ # TODO next token?
+ case @lex_state
+ when EXPR_FNAME, EXPR_DOT
+ @lex_state = EXPR_ARG
+ else
+ @lex_state = EXPR_BEG
+ end
+ when '?'
+ @lex_state = EXPR_BEG
+ when '&&', '||', '+=', '-=', '*=', '**=',
+ '&=', '|=', '^=', '<<=', '>>=', '||=', '&&='
+ @lex_state = EXPR_BEG
+ else
+ case @lex_state
+ when EXPR_FNAME, EXPR_DOT
+ @lex_state = EXPR_ARG
+ else
+ @lex_state = EXPR_BEG
+ end
+ end
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_kw(tok, data)
+ case tok
+ when 'class'
+ @lex_state = EXPR_CLASS
+ @in_fname = true
+ when 'def'
+ @lex_state = EXPR_FNAME
+ @continue = true
+ @in_fname = true
+ when 'if', 'unless', 'while', 'until'
+ if ((EXPR_END | EXPR_ENDARG | EXPR_ENDFN | EXPR_ARG | EXPR_CMDARG) & @lex_state) != 0 # postfix if
+ @lex_state = EXPR_BEG | EXPR_LABEL
+ else
+ @lex_state = EXPR_BEG
+ end
+ when 'begin'
+ @lex_state = EXPR_BEG
+ else
+ if @lex_state == EXPR_FNAME
+ @lex_state = EXPR_END
+ else
+ @lex_state = EXPR_END
+ end
+ end
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_tstring_beg(tok, data)
+ @lex_state = EXPR_BEG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_tstring_end(tok, data)
+ @lex_state = EXPR_END | EXPR_ENDARG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_CHAR(tok, data)
+ @lex_state = EXPR_END
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_period(tok, data)
+ @lex_state = EXPR_DOT
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_int(tok, data)
+ @lex_state = EXPR_END | EXPR_ENDARG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_float(tok, data)
+ @lex_state = EXPR_END | EXPR_ENDARG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_rational(tok, data)
+ @lex_state = EXPR_END | EXPR_ENDARG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_imaginary(tok, data)
+ @lex_state = EXPR_END | EXPR_ENDARG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_symbeg(tok, data)
+ @lex_state = EXPR_FNAME
+ @continue = true
+ @in_fname = true
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ private def on_variables(event, tok, data)
+ if @in_fname
+ @lex_state = EXPR_ENDFN
+ @in_fname = false
+ @continue = false
+ elsif @continue
+ case @lex_state
+ when EXPR_DOT
+ @lex_state = EXPR_ARG
+ else
+ @lex_state = EXPR_ENDFN
+ @continue = false
+ end
+ else
+ @lex_state = EXPR_CMDARG
+ end
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state})
+ end
+
+ def on_ident(tok, data)
+ on_variables(__method__, tok, data)
+ end
+
+ def on_ivar(tok, data)
+ @lex_state = EXPR_END
+ on_variables(__method__, tok, data)
+ end
+
+ def on_cvar(tok, data)
+ @lex_state = EXPR_END
+ on_variables(__method__, tok, data)
+ end
+
+ def on_gvar(tok, data)
+ @lex_state = EXPR_END
+ on_variables(__method__, tok, data)
+ end
+
+ def on_backref(tok, data)
+ @lex_state = EXPR_END
+ on_variables(__method__, tok, data)
+ end
+
+ def on_lparen(tok, data)
+ @lex_state = EXPR_LABEL | EXPR_BEG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_rparen(tok, data)
+ @lex_state = EXPR_ENDFN
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_lbrace(tok, data)
+ @lex_state = EXPR_LABEL | EXPR_BEG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_rbrace(tok, data)
+ @lex_state = EXPR_ENDARG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_lbracket(tok, data)
+ @lex_state = EXPR_LABEL | EXPR_BEG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_rbracket(tok, data)
+ @lex_state = EXPR_ENDARG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_const(tok, data)
+ case @lex_state
+ when EXPR_FNAME
+ @lex_state = EXPR_ENDFN
+ when EXPR_CLASS
+ @lex_state = EXPR_ARG
+ else
+ @lex_state = EXPR_CMDARG
+ end
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_sp(tok, data)
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_comma(tok, data)
+ @lex_state = EXPR_BEG | EXPR_LABEL if (EXPR_ARG_ANY & @lex_state) != 0
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_comment(tok, data)
+ @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_ignored_sp(tok, data)
+ @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_heredoc_end(tok, data)
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ @lex_state = EXPR_BEG
+ end
+
+ def on_default(event, tok, data)
+ reset
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state})
+ end
+
+ def each(&block)
+ @callback = block
+ parse
+ end
+ end
+
+ def get_squashed_tk
+ if @buf.empty?
+ tk = @inner_lex_enumerator.next
+ else
+ tk = @buf.shift
+ end
+ case tk[:kind]
+ when :on_symbeg then
+ tk = get_symbol_tk(tk)
+ when :on_tstring_beg then
+ tk = get_string_tk(tk)
+ when :on_backtick then
+ if (EXPR_FNAME & tk[:state]) != 0
+ @inner_lex.lex_state = EXPR_ARG
+ tk[:kind] = :on_ident
+ tk[:state] = @inner_lex.lex_state
+ else
+ tk = get_string_tk(tk)
+ end
+ when :on_regexp_beg then
+ tk = get_regexp_tk(tk)
+ when :on_embdoc_beg then
+ tk = get_embdoc_tk(tk)
+ when :on_heredoc_beg then
+ @heredoc_queue << retrieve_heredoc_info(tk)
+ @inner_lex.lex_state = EXPR_END
+ when :on_nl, :on_ignored_nl, :on_comment, :on_heredoc_end then
+ unless @heredoc_queue.empty?
+ get_heredoc_tk(*@heredoc_queue.shift)
+ end
+ when :on_words_beg then
+ tk = get_words_tk(tk)
+ when :on_qwords_beg then
+ tk = get_words_tk(tk)
+ when :on_symbols_beg then
+ tk = get_words_tk(tk)
+ when :on_qsymbols_beg then
+ tk = get_words_tk(tk)
+ when :on_op then
+ if '&.' == tk[:text]
+ tk[:kind] = :on_period
+ else
+ tk = get_op_tk(tk)
+ end
+ end
+ tk
+ end
+
+ private def get_symbol_tk(tk)
+ is_symbol = true
+ symbol_tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_symbol }
+ if ":'" == tk[:text] or ':"' == tk[:text]
+ tk1 = get_string_tk(tk)
+ symbol_tk[:text] = tk1[:text]
+ symbol_tk[:state] = tk1[:state]
+ else
+ case (tk1 = get_squashed_tk)[:kind]
+ when :on_ident
+ symbol_tk[:text] = ":#{tk1[:text]}"
+ symbol_tk[:state] = tk1[:state]
+ when :on_tstring_content
+ symbol_tk[:text] = ":#{tk1[:text]}"
+ symbol_tk[:state] = get_squashed_tk[:state] # skip :on_tstring_end
+ when :on_tstring_end
+ symbol_tk[:text] = ":#{tk1[:text]}"
+ symbol_tk[:state] = tk1[:state]
+ when :on_op
+ symbol_tk[:text] = ":#{tk1[:text]}"
+ symbol_tk[:state] = tk1[:state]
+ when :on_ivar
+ symbol_tk[:text] = ":#{tk1[:text]}"
+ symbol_tk[:state] = tk1[:state]
+ when :on_cvar
+ symbol_tk[:text] = ":#{tk1[:text]}"
+ symbol_tk[:state] = tk1[:state]
+ when :on_gvar
+ symbol_tk[:text] = ":#{tk1[:text]}"
+ symbol_tk[:state] = tk1[:state]
+ when :on_const
+ symbol_tk[:text] = ":#{tk1[:text]}"
+ symbol_tk[:state] = tk1[:state]
+ when :on_kw
+ symbol_tk[:text] = ":#{tk1[:text]}"
+ symbol_tk[:state] = tk1[:state]
+ else
+ is_symbol = false
+ tk = tk1
+ end
+ end
+ if is_symbol
+ tk = symbol_tk
+ end
+ tk
+ end
+
+ private def get_string_tk(tk)
+ string = tk[:text]
+ state = nil
+ kind = :on_tstring
+ loop do
+ inner_str_tk = get_squashed_tk
+ if inner_str_tk.nil?
+ break
+ elsif :on_tstring_end == inner_str_tk[:kind]
+ string = string + inner_str_tk[:text]
+ state = inner_str_tk[:state]
+ break
+ elsif :on_label_end == inner_str_tk[:kind]
+ string = string + inner_str_tk[:text]
+ state = inner_str_tk[:state]
+ kind = :on_symbol
+ break
+ else
+ string = string + inner_str_tk[:text]
+ if :on_embexpr_beg == inner_str_tk[:kind] then
+ kind = :on_dstring if :on_tstring == kind
+ end
+ end
+ end
+ {
+ :line_no => tk[:line_no],
+ :char_no => tk[:char_no],
+ :kind => kind,
+ :text => string,
+ :state => state
+ }
+ end
+
+ private def get_regexp_tk(tk)
+ string = tk[:text]
+ state = nil
+ loop do
+ inner_str_tk = get_squashed_tk
+ if inner_str_tk.nil?
+ break
+ elsif :on_regexp_end == inner_str_tk[:kind]
+ string = string + inner_str_tk[:text]
+ state = inner_str_tk[:state]
+ break
+ else
+ string = string + inner_str_tk[:text]
+ end
+ end
+ {
+ :line_no => tk[:line_no],
+ :char_no => tk[:char_no],
+ :kind => :on_regexp,
+ :text => string,
+ :state => state
+ }
+ end
+
+ private def get_embdoc_tk(tk)
+ string = tk[:text]
+ until :on_embdoc_end == (embdoc_tk = get_squashed_tk)[:kind] do
+ string = string + embdoc_tk[:text]
+ end
+ string = string + embdoc_tk[:text]
+ {
+ :line_no => tk[:line_no],
+ :char_no => tk[:char_no],
+ :kind => :on_embdoc,
+ :text => string,
+ :state => embdoc_tk[:state]
+ }
+ end
+
+ private def get_heredoc_tk(heredoc_name, indent)
+ string = ''
+ start_tk = nil
+ prev_tk = nil
+ until heredoc_end?(heredoc_name, indent, tk = @inner_lex_enumerator.next) do
+ start_tk = tk unless start_tk
+ if (prev_tk.nil? or "\n" == prev_tk[:text][-1]) and 0 != tk[:char_no]
+ string = string + (' ' * tk[:char_no])
+ end
+ string = string + tk[:text]
+ prev_tk = tk
+ end
+ start_tk = tk unless start_tk
+ prev_tk = tk unless prev_tk
+ @buf.unshift tk # closing heredoc
+ heredoc_tk = {
+ :line_no => start_tk[:line_no],
+ :char_no => start_tk[:char_no],
+ :kind => :on_heredoc,
+ :text => string,
+ :state => prev_tk[:state]
+ }
+ @buf.unshift heredoc_tk
+ end
+
+ private def retrieve_heredoc_info(tk)
+ name = tk[:text].gsub(/\A<<[-~]?(['"`]?)(.+)\1\z/, '\2')
+ indent = tk[:text] =~ /\A<<[-~]/
+ [name, indent]
+ end
+
+ private def heredoc_end?(name, indent, tk)
+ result = false
+ if :on_heredoc_end == tk[:kind] then
+ tk_name = (indent ? tk[:text].gsub(/^ *(.+)\n?$/, '\1') : tk[:text].gsub(/\n\z/, ''))
+ if name == tk_name
+ result = true
+ end
+ end
+ result
+ end
+
+ private def get_words_tk(tk)
+ string = ''
+ start_token = tk[:text]
+ start_quote = tk[:text].rstrip[-1]
+ line_no = tk[:line_no]
+ char_no = tk[:char_no]
+ state = tk[:state]
+ end_quote =
+ case start_quote
+ when ?( then ?)
+ when ?[ then ?]
+ when ?{ then ?}
+ when ?< then ?>
+ else start_quote
+ end
+ end_token = nil
+ loop do
+ tk = get_squashed_tk
+ if tk.nil?
+ end_token = end_quote
+ break
+ elsif :on_tstring_content == tk[:kind] then
+ string += tk[:text]
+ elsif :on_words_sep == tk[:kind] or :on_tstring_end == tk[:kind] then
+ if end_quote == tk[:text].strip then
+ end_token = tk[:text]
+ break
+ else
+ string += tk[:text]
+ end
+ else
+ string += tk[:text]
+ end
+ end
+ text = "#{start_token}#{string}#{end_token}"
+ {
+ :line_no => line_no,
+ :char_no => char_no,
+ :kind => :on_dstring,
+ :text => text,
+ :state => state
+ }
+ end
+
+ private def get_op_tk(tk)
+ redefinable_operators = %w[! != !~ % & * ** + +@ - -@ / < << <= <=> == === =~ > >= >> [] []= ^ ` | ~]
+ if redefinable_operators.include?(tk[:text]) and EXPR_ARG == tk[:state] then
+ @inner_lex.lex_state = EXPR_ARG
+ tk[:kind] = :on_ident
+ tk[:state] = @inner_lex.lex_state
+ elsif tk[:text] =~ /^[-+]$/ then
+ tk_ahead = get_squashed_tk
+ case tk_ahead[:kind]
+ when :on_int, :on_float, :on_rational, :on_imaginary then
+ tk[:text] += tk_ahead[:text]
+ tk[:kind] = tk_ahead[:kind]
+ tk[:state] = tk_ahead[:state]
+ else
+ @buf.unshift tk_ahead
+ end
+ end
+ tk
+ end
+
+ def initialize(code)
+ @buf = []
+ @heredoc_queue = []
+ @inner_lex = InnerStateLex.new(code)
+ @inner_lex_enumerator = Enumerator.new do |y|
+ @inner_lex.each do |tk|
+ y << tk
+ end
+ end
+ end
+
+ def self.parse(code)
+ lex = self.new(code)
+ tokens = []
+ begin
+ while tk = lex.get_squashed_tk
+ tokens.push tk
+ end
+ rescue StopIteration
+ end
+ tokens
+ end
+
+ def self.end?(token)
+ (token[:state] & EXPR_END)
+ end
+end
diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb
index e7900c0807..130eca89c7 100644
--- a/lib/rdoc/parser/ruby.rb
+++ b/lib/rdoc/parser/ruby.rb
@@ -24,6 +24,7 @@ $TOKEN_DEBUG ||= nil
# * aliases
# * private, public, protected
# * private_class_function, public_class_function
+# * private_constant, public_constant
# * module_function
# * attr, attr_reader, attr_writer, attr_accessor
# * extra accessors given on the command line
@@ -139,11 +140,12 @@ $TOKEN_DEBUG ||= nil
# Note that by default, the :method: directive will be ignored if there is a
# standard rdocable item following it.
+require 'ripper'
+
class RDoc::Parser::Ruby < RDoc::Parser
parse_files_matching(/\.rbw?$/)
- include RDoc::RubyToken
include RDoc::TokenStream
include RDoc::Parser::RubyTools
@@ -163,10 +165,20 @@ class RDoc::Parser::Ruby < RDoc::Parser
def initialize(top_level, file_name, content, options, stats)
super
+ if /\t/ =~ content then
+ tab_width = @options.tab_width
+ content = content.split(/\n/).map do |line|
+ 1 while line.gsub!(/\t+/) {
+ ' ' * (tab_width*$&.length - $`.length % tab_width)
+ } && $~
+ line
+ end.join("\n")
+ end
+
@size = 0
@token_listeners = nil
- @scanner = RDoc::RubyLex.new content, @options
- @scanner.exception_on_syntax_error = false
+ @scanner = RDoc::RipperStateLex.parse(content)
+ @scanner_point = 0
@prev_seek = nil
@markup = @options.markup
@track_visibility = :nodoc != @options.visibility
@@ -175,6 +187,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
reset
end
+ def tk_nl?(tk)
+ :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]
+ end
+
##
# Retrieves the read token stream and replaces +pattern+ with +replacement+
# using gsub. If the result is only a ";" returns an empty string.
@@ -194,7 +210,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
# methods.
def get_visibility_information tk, single # :nodoc:
- vis_type = tk.name
+ vis_type = tk[:text]
singleton = single == SINGLE
vis =
@@ -226,27 +242,27 @@ class RDoc::Parser::Ruby < RDoc::Parser
comment = ''
comment.force_encoding @encoding if @encoding
first_line = true
- first_comment_tk_class = nil
+ first_comment_tk_kind = nil
tk = get_tk
- while TkCOMMENT === tk
- if first_line and tk.text =~ /\A#!/ then
+ while tk && (:on_comment == tk[:kind] or :on_embdoc == tk[:kind])
+ if first_line and tk[:text] =~ /\A#!/ then
skip_tkspace
tk = get_tk
- elsif first_line and tk.text =~ /\A#\s*-\*-/ then
+ elsif first_line and tk[:text] =~ /\A#\s*-\*-/ then
first_line = false
skip_tkspace
tk = get_tk
else
- break if first_comment_tk_class and not first_comment_tk_class === tk
- first_comment_tk_class = tk.class
+ break if first_comment_tk_kind and not first_comment_tk_kind === tk[:kind]
+ first_comment_tk_kind = tk[:kind]
first_line = false
- comment << tk.text << "\n"
+ comment << tk[:text]
tk = get_tk
- if TkNL === tk then
+ if :on_nl === tk then
skip_tkspace false
tk = get_tk
end
@@ -262,7 +278,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
# Consumes trailing whitespace from the token stream
def consume_trailing_spaces # :nodoc:
- get_tkread
skip_tkspace false
end
@@ -309,10 +324,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
def get_bool
skip_tkspace
tk = get_tk
- case tk
- when TkTRUE
+ if :on_kw == tk[:kind] && 'true' == tk[:text]
true
- when TkFALSE, TkNIL
+ elsif :on_kw == tk[:kind] && ('false' == tk[:text] || 'nil' == tk[:text])
false
else
unget_tk tk
@@ -331,24 +345,24 @@ class RDoc::Parser::Ruby < RDoc::Parser
given_name = ''
# class ::A -> A is in the top level
- case name_t
- when TkCOLON2, TkCOLON3 then # bug
+ if :on_op == name_t[:kind] and '::' == name_t[:text] then # bug
name_t = get_tk
container = @top_level
given_name << '::'
end
skip_tkspace false
- given_name << name_t.name
+ given_name << name_t[:text]
- while TkCOLON2 === peek_tk do
+ is_self = name_t[:kind] == :on_op && name_t[:text] == '<<'
+ while !is_self && (tk = peek_tk) and :on_op == tk[:kind] and '::' == tk[:text] do
prev_container = container
- container = container.find_module_named name_t.name
+ container = container.find_module_named name_t[:text]
container ||=
if ignore_constants then
RDoc::Context.new
else
- c = prev_container.add_module RDoc::NormalModule, name_t.name
+ c = prev_container.add_module RDoc::NormalModule, name_t[:text]
c.ignore unless prev_container.document_children
@top_level.add_to_classes_or_modules c
c
@@ -359,7 +373,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
get_tk
skip_tkspace false
name_t = get_tk
- given_name << '::' << name_t.name
+ unless :on_const == name_t[:kind] || :on_ident == name_t[:kind]
+ raise RDoc::Error, "Invalid class or module definition: #{given_name}"
+ end
+ given_name << '::' << name_t[:text]
end
skip_tkspace false
@@ -371,9 +388,13 @@ class RDoc::Parser::Ruby < RDoc::Parser
# Return a superclass, which can be either a constant of an expression
def get_class_specification
- case peek_tk
- when TkSELF then return 'self'
- when TkGVAR then return ''
+ tk = peek_tk
+ if tk.nil?
+ return ''
+ elsif :on_kw == tk[:kind] && 'self' == tk[:text]
+ return 'self'
+ elsif :on_gvar == tk[:kind]
+ return ''
end
res = get_constant
@@ -383,9 +404,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
get_tkread # empty out read buffer
tk = get_tk
+ return res unless tk
- case tk
- when TkNL, TkCOMMENT, TkSEMICOLON then
+ case tk[:kind]
+ when :on_nl, :on_comment, :on_embdoc, :on_semicolon then
unget_tk(tk)
return res
end
@@ -403,8 +425,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
skip_tkspace false
tk = get_tk
- while TkCOLON2 === tk or TkCOLON3 === tk or TkCONSTANT === tk do
- res += tk.name
+ while tk && ((:on_op == tk[:kind] && '::' == tk[:text]) || :on_const == tk[:kind]) do
+ res += tk[:text]
tk = get_tk
end
@@ -420,7 +442,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
nest = 0
- while TkLPAREN === (tk = peek_tk) or TkfLPAREN === tk do
+ while :on_lparen == (tk = peek_tk)[:kind] do
get_tk
skip_tkspace
nest += 1
@@ -431,7 +453,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
while nest > 0
skip_tkspace
tk = get_tk
- nest -= 1 if TkRPAREN === tk
+ nest -= 1 if :on_rparen == tk[:kind]
end
name
@@ -446,13 +468,19 @@ class RDoc::Parser::Ruby < RDoc::Parser
# won't catch all cases (such as "a = yield + 1"
def get_end_token tk # :nodoc:
- case tk
- when TkLPAREN, TkfLPAREN
- TkRPAREN
- when TkRPAREN
+ case tk[:kind]
+ when :on_lparen
+ {
+ :kind => :on_rparen,
+ :text => ')'
+ }
+ when :on_rparen
nil
else
- TkNL
+ {
+ :kind => :on_nl,
+ :text => "\n"
+ }
end
end
@@ -461,11 +489,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
def get_method_container container, name_t # :nodoc:
prev_container = container
- container = container.find_module_named(name_t.name)
+ container = container.find_module_named(name_t[:text])
unless container then
constant = prev_container.constants.find do |const|
- const.name == name_t.name
+ const.name == name_t[:text]
end
if constant then
@@ -476,21 +504,21 @@ class RDoc::Parser::Ruby < RDoc::Parser
unless container then
# TODO seems broken, should starting at Object in @store
- obj = name_t.name.split("::").inject(Object) do |state, item|
+ obj = name_t[:text].split("::").inject(Object) do |state, item|
state.const_get(item)
end rescue nil
type = obj.class == Class ? RDoc::NormalClass : RDoc::NormalModule
unless [Class, Module].include?(obj.class) then
- warn("Couldn't find #{name_t.name}. Assuming it's a module")
+ warn("Couldn't find #{name_t[:text]}. Assuming it's a module")
end
if type == RDoc::NormalClass then
sclass = obj.superclass ? obj.superclass.name : nil
- container = prev_container.add_class type, name_t.name, sclass
+ container = prev_container.add_class type, name_t[:text], sclass
else
- container = prev_container.add_module type, name_t.name
+ container = prev_container.add_module type, name_t[:text]
end
record_location container
@@ -504,32 +532,26 @@ class RDoc::Parser::Ruby < RDoc::Parser
def get_symbol_or_name
tk = get_tk
- case tk
- when TkSYMBOL then
- text = tk.text.sub(/^:/, '')
+ case tk[:kind]
+ when :on_symbol then
+ text = tk[:text].sub(/^:/, '')
- if TkASSIGN === peek_tk then
+ next_tk = peek_tk
+ if next_tk && :on_op == next_tk[:kind] && '=' == next_tk[:text] then
get_tk
text << '='
end
text
- when TkId, TkOp then
- tk.name
- when TkAMPER,
- TkDSTRING,
- TkSTAR,
- TkSTRING then
- tk.text
+ when :on_ident, :on_const, :on_gvar, :on_cvar, :on_ivar, :on_op, :on_kw then
+ tk[:text]
+ when :on_tstring, :on_dstring then
+ tk[:text][1..-2]
else
raise RDoc::Error, "Name or symbol expected (got #{tk})"
end
end
- def stop_at_EXPR_END # :nodoc:
- @scanner.lex_state == :EXPR_END || !@scanner.continue
- end
-
##
# Marks containers between +container+ and +ancestor+ as ignored
@@ -570,7 +592,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
def make_message message
prefix = "#{@file_name}:"
- prefix << "#{@scanner.line_no}:#{@scanner.char_no}:" if @scanner
+ tk = peek_tk
+ prefix << "#{tk[:line_no]}:#{tk[:char_no]}:" if tk
"#{prefix} #{message}"
end
@@ -589,7 +612,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
# +comment+.
def parse_attr(context, single, tk, comment)
- line_no = tk.line_no
+ line_no = tk[:line_no]
args = parse_symbol_arg 1
if args.size > 0 then
@@ -598,7 +621,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
skip_tkspace false
tk = get_tk
- if TkCOMMA === tk then
+ if :on_comma == tk[:kind] then
rw = "RW" if get_bool
else
unget_tk tk
@@ -618,7 +641,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
# comment for each to +comment+.
def parse_attr_accessor(context, single, tk, comment)
- line_no = tk.line_no
+ line_no = tk[:line_no]
args = parse_symbol_arg
rw = "?"
@@ -629,7 +652,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
# and add found items appropriately but here we do not. I'm not sure why.
return if @track_visibility and not tmp.document_self
- case tk.name
+ case tk[:text]
when "attr_reader" then rw = "R"
when "attr_writer" then rw = "W"
when "attr_accessor" then rw = "RW"
@@ -647,21 +670,19 @@ class RDoc::Parser::Ruby < RDoc::Parser
# Parses an +alias+ in +context+ with +comment+
def parse_alias(context, single, tk, comment)
- line_no = tk.line_no
+ line_no = tk[:line_no]
skip_tkspace
- if TkLPAREN === peek_tk then
+ if :on_lparen === peek_tk[:kind] then
get_tk
skip_tkspace
end
new_name = get_symbol_or_name
- @scanner.lex_state = :EXPR_FNAME
-
skip_tkspace
- if TkCOMMA === peek_tk then
+ if :on_comma === peek_tk[:kind] then
get_tk
skip_tkspace
end
@@ -688,34 +709,38 @@ class RDoc::Parser::Ruby < RDoc::Parser
# Extracts call parameters from the token stream.
def parse_call_parameters(tk)
- end_token = case tk
- when TkLPAREN, TkfLPAREN
- TkRPAREN
- when TkRPAREN
+ end_token = case tk[:kind]
+ when :on_lparen
+ :on_rparen
+ when :on_rparen
return ""
else
- TkNL
+ :on_nl
end
nest = 0
loop do
- case tk
- when TkSEMICOLON
+ break if tk.nil?
+ case tk[:kind]
+ when :on_semicolon
break
- when TkLPAREN, TkfLPAREN
+ when :on_lparen
nest += 1
when end_token
- if end_token == TkRPAREN
+ if end_token == :on_rparen
nest -= 1
- break if @scanner.lex_state == :EXPR_END and nest <= 0
+ break if RDoc::RipperStateLex.end?(tk) and nest <= 0
else
- break unless @scanner.continue
+ break if RDoc::RipperStateLex.end?(tk)
end
- when TkCOMMENT, TkASSIGN, TkOPASGN
+ when :on_comment, :on_embdoc
unget_tk(tk)
break
- when nil then
- break
+ when :on_op
+ if tk[:text] =~ /^(.{1,2})?=$/
+ unget_tk(tk)
+ break
+ end
end
tk = get_tk
end
@@ -727,28 +752,27 @@ class RDoc::Parser::Ruby < RDoc::Parser
# Parses a class in +context+ with +comment+
def parse_class container, single, tk, comment
- line_no = tk.line_no
+ line_no = tk[:line_no]
declaration_context = container
container, name_t, given_name = get_class_or_module container
- cls =
- case name_t
- when TkCONSTANT
- parse_class_regular container, declaration_context, single,
- name_t, given_name, comment
- when TkLSHFT
- case name = get_class_specification
- when 'self', container.name
- parse_statements container, SINGLE
- return # don't update line
- else
- parse_class_singleton container, name, comment
- end
+ if name_t[:kind] == :on_const
+ cls = parse_class_regular container, declaration_context, single,
+ name_t, given_name, comment
+ elsif name_t[:kind] == :on_op && name_t[:text] == '<<'
+ case name = get_class_specification
+ when 'self', container.name
+ read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
+ parse_statements container, SINGLE
+ return # don't update line
else
- warn "Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}"
- return
+ cls = parse_class_singleton container, name, comment
end
+ else
+ warn "Expected class name or '<<'. Got #{name_t[:kind]}: #{name_t[:text].inspect}"
+ return
+ end
cls.line = line_no
@@ -770,7 +794,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
given_name = $'
end
- if TkLT === peek_tk then
+ tk = peek_tk
+ if tk[:kind] == :on_op && tk[:text] == '<' then
get_tk
skip_tkspace
superclass = get_class_specification
@@ -840,40 +865,52 @@ class RDoc::Parser::Ruby < RDoc::Parser
# true, no found constants will be added to RDoc.
def parse_constant container, tk, comment, ignore_constants = false
- line_no = tk.line_no
+ line_no = tk[:line_no]
- name = tk.name
+ name = tk[:text]
skip_tkspace false
return unless name =~ /^\w+$/
eq_tk = get_tk
- if TkCOLON2 === eq_tk then
+ if :on_op == eq_tk[:kind] && '::' == eq_tk[:text] then
unget_tk eq_tk
unget_tk tk
container, name_t, = get_class_or_module container, ignore_constants
- name = name_t.name
+ name = name_t[:text]
eq_tk = get_tk
end
- unless TkASSIGN === eq_tk then
- unget_tk eq_tk
- return false
+ is_array_or_hash = false
+ if eq_tk && :on_lbracket == eq_tk[:kind]
+ nest = 1
+ while bracket_tk = get_tk
+ case bracket_tk[:kind]
+ when :on_lbracket
+ nest += 1
+ when :on_rbracket
+ nest -= 1
+ break if nest == 0
+ end
+ end
+ skip_tkspace false
+ eq_tk = get_tk
+ is_array_or_hash = true
end
- if TkGT === peek_tk then
+ unless eq_tk && :on_op == eq_tk[:kind] && '=' == eq_tk[:text] then
unget_tk eq_tk
- return
+ return false
end
value = ''
con = RDoc::Constant.new name, value, comment
- body = parse_constant_body container, con
+ body = parse_constant_body container, con, is_array_or_hash
return unless body
@@ -882,13 +919,15 @@ class RDoc::Parser::Ruby < RDoc::Parser
con.line = line_no
read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS
+ return if is_array_or_hash
+
@stats.add_constant con
container.add_constant con
true
end
- def parse_constant_body container, constant # :nodoc:
+ def parse_constant_body container, constant, is_array_or_hash # :nodoc:
nest = 0
rhs_name = ''
@@ -896,44 +935,51 @@ class RDoc::Parser::Ruby < RDoc::Parser
tk = get_tk
+ body = nil
loop do
- case tk
- when TkSEMICOLON then
+ break if tk.nil?
+ if :on_semicolon == tk[:kind] then
break if nest <= 0
- when TkLPAREN, TkfLPAREN, TkLBRACE, TkfLBRACE, TkLBRACK, TkfLBRACK,
- TkDO, TkIF, TkUNLESS, TkCASE, TkDEF, TkBEGIN then
+ elsif [:on_tlambeg, :on_lparen, :on_lbrace, :on_lbracket].include?(tk[:kind]) then
nest += 1
- when TkRPAREN, TkRBRACE, TkRBRACK, TkEND then
+ elsif (:on_kw == tk[:kind] && 'def' == tk[:text]) then
+ nest += 1
+ elsif (:on_kw == tk[:kind] && %w{do if unless case begin}.include?(tk[:text])) then
+ if (RDoc::RipperStateLex::EXPR_LABEL & tk[:state]) == 0
+ nest += 1
+ end
+ elsif [:on_rparen, :on_rbrace, :on_rbracket].include?(tk[:kind]) ||
+ (:on_kw == tk[:kind] && 'end' == tk[:text]) then
nest -= 1
- when TkCOMMENT then
- if nest <= 0 and stop_at_EXPR_END then
- unget_tk tk
+ elsif (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then
+ unget_tk tk
+ if nest <= 0 and RDoc::RipperStateLex.end?(tk) then
+ body = get_tkread_clean(/^[ \t]+/, '')
+ read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS
break
else
- unget_tk tk
read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS
end
- when TkCONSTANT then
- rhs_name << tk.name
+ elsif :on_const == tk[:kind] then
+ rhs_name << tk[:text]
- if nest <= 0 and TkNL === peek_tk then
- create_module_alias container, constant, rhs_name
+ next_tk = peek_tk
+ if nest <= 0 and (next_tk.nil? || :on_nl == next_tk[:kind]) then
+ create_module_alias container, constant, rhs_name unless is_array_or_hash
break
end
- when TkNL then
- if nest <= 0 and stop_at_EXPR_END then
+ elsif :on_nl == tk[:kind] then
+ if nest <= 0 and RDoc::RipperStateLex.end?(tk) then
unget_tk tk
break
end
- when TkCOLON2, TkCOLON3 then
+ elsif :on_op == tk[:kind] && '::' == tk[:text]
rhs_name << '::'
- when nil then
- break
end
tk = get_tk
end
- get_tkread_clean(/^[ \t]+/, '')
+ body ? body : get_tkread_clean(/^[ \t]+/, '')
end
##
@@ -942,8 +988,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
def parse_comment container, tk, comment
return parse_comment_tomdoc container, tk, comment if @markup == 'tomdoc'
- column = tk.char_no
- line_no = tk.line_no
+ column = tk[:char_no]
+ line_no = tk[:line_no]
text = comment.text
@@ -988,12 +1034,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
record_location meth
meth.start_collecting_tokens
- indent = TkSPACE.new 0, 1, 1
- indent.set_text " " * column
-
- position_comment = TkCOMMENT.new 0, line_no, 1
- position_comment.set_text "# File #{@top_level.relative_name}, line #{line_no}"
- meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
+ indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
+ position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
+ position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
+ newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
+ meth.add_tokens [position_comment, newline, indent]
meth.params =
if text.sub!(/^#\s+:?args?:\s*(.*?)\s*$/i, '') then
@@ -1022,8 +1067,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
def parse_comment_tomdoc container, tk, comment
return unless signature = RDoc::TomDoc.signature(comment)
- column = tk.char_no
- line_no = tk.line_no
+ column = tk[:char_no]
+ line_no = tk[:line_no]
name, = signature.split %r%[ \(]%, 2
@@ -1032,12 +1077,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
meth.line = line_no
meth.start_collecting_tokens
- indent = TkSPACE.new 0, 1, 1
- indent.set_text " " * column
-
- position_comment = TkCOMMENT.new 0, line_no, 1
- position_comment.set_text "# File #{@top_level.relative_name}, line #{line_no}"
- meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
+ indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
+ position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
+ position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
+ newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
+ meth.add_tokens [position_comment, newline, indent]
meth.call_seq = signature
@@ -1067,7 +1111,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
record_location obj
end
- return unless TkCOMMA === peek_tk
+ return if peek_tk.nil? || :on_comma != peek_tk[:kind]
get_tk
end
@@ -1079,11 +1123,14 @@ class RDoc::Parser::Ruby < RDoc::Parser
# Returns true if the comment was not consumed.
def parse_identifier container, single, tk, comment # :nodoc:
- case tk.name
+ case tk[:text]
when 'private', 'protected', 'public', 'private_class_method',
'public_class_method', 'module_function' then
parse_visibility container, single, tk
return true
+ when 'private_constant', 'public_constant'
+ parse_constant_visibility container, single, tk
+ return true
when 'attr' then
parse_attr container, single, tk, comment
when /^attr_(reader|writer|accessor)$/ then
@@ -1172,8 +1219,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
# Parses a meta-programmed method
def parse_meta_method(container, single, tk, comment)
- column = tk.char_no
- line_no = tk.line_no
+ column = tk[:char_no]
+ line_no = tk[:line_no]
start_collecting_tokens
add_token tk
@@ -1195,12 +1242,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
remove_token_listener self
meth.start_collecting_tokens
- indent = TkSPACE.new 0, 1, 1
- indent.set_text " " * column
-
- position_comment = TkCOMMENT.new 0, line_no, 1
- position_comment.value = "# File #{@top_level.relative_name}, line #{line_no}"
- meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
+ indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
+ position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
+ position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
+ newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
+ meth.add_tokens [position_comment, newline, indent]
meth.add_tokens @token_stream
parse_meta_method_params container, single, meth, tk, comment
@@ -1224,17 +1270,16 @@ class RDoc::Parser::Ruby < RDoc::Parser
name_t = get_tk
- case name_t
- when TkSYMBOL then
- name_t.text[1..-1]
- when TkSTRING then
- name_t.value[1..-2]
- when TkASSIGN then # ignore
+ if :on_symbol == name_t[:kind] then
+ name_t[:text][1..-1]
+ elsif :on_tstring == name_t[:kind] then
+ name_t[:text][1..-2]
+ elsif :on_op == name_t[:kind] && '=' == name_t[:text] then # ignore
remove_token_listener self
nil
else
- warn "unknown name token #{name_t.inspect} for meta-method '#{tk.name}'"
+ warn "unknown name token #{name_t.inspect} for meta-method '#{tk[:text]}'"
'unknown'
end
end
@@ -1254,14 +1299,13 @@ class RDoc::Parser::Ruby < RDoc::Parser
last_tk = tk
while tk = get_tk do
- case tk
- when TkSEMICOLON then
+ if :on_semicolon == tk[:kind] then
break
- when TkNL then
- break unless last_tk and TkCOMMA === last_tk
- when TkSPACE then
+ elsif :on_nl == tk[:kind] then
+ break unless last_tk and :on_comma == last_tk[:kind]
+ elsif :on_sp == tk[:kind] then
# expression continues
- when TkDO then
+ elsif :on_kw == tk[:kind] && 'do' == tk[:text] then
parse_statements container, single, meth
break
else
@@ -1278,8 +1322,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
singleton = nil
added_container = false
name = nil
- column = tk.char_no
- line_no = tk.line_no
+ column = tk[:char_no]
+ line_no = tk[:line_no]
start_collecting_tokens
add_token tk
@@ -1299,12 +1343,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
meth.line = line_no
meth.start_collecting_tokens
- indent = TkSPACE.new 0, 1, 1
- indent.set_text " " * column
-
- token = TkCOMMENT.new 0, line_no, 1
- token.set_text "# File #{@top_level.relative_name}, line #{line_no}"
- meth.add_tokens [token, NEWLINE_TOKEN, indent]
+ indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
+ token = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
+ token[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
+ newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
+ meth.add_tokens [token, newline, indent]
meth.add_tokens @token_stream
parse_method_params_and_body container, single, meth, added_container
@@ -1325,7 +1368,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
def parse_method_params_and_body container, single, meth, added_container
token_listener meth do
- @scanner.continue = false
parse_method_parameters meth
if meth.document_self or not @track_visibility then
@@ -1368,15 +1410,13 @@ class RDoc::Parser::Ruby < RDoc::Parser
# it is a singleton or regular method.
def parse_method_name container # :nodoc:
- @scanner.lex_state = :EXPR_FNAME
-
skip_tkspace
name_t = get_tk
- back_tk = skip_tkspace
+ back_tk = skip_tkspace(false)
singleton = false
- case dot = get_tk
- when TkDOT, TkCOLON2 then
+ dot = get_tk
+ if dot[:kind] == :on_period || (dot[:kind] == :on_op && dot[:text] == '::') then
singleton = true
name, container = parse_method_name_singleton container, name_t
@@ -1397,16 +1437,15 @@ class RDoc::Parser::Ruby < RDoc::Parser
# is parsed from the token stream for a regular method.
def parse_method_name_regular container, name_t # :nodoc:
- case name_t
- when TkSTAR, TkAMPER then
- name_t.text
+ if :on_op == name_t[:kind] && (%w{* & [] []= <<}.include?(name_t[:text])) then
+ name_t[:text]
else
- unless name_t.respond_to? :name then
+ unless [:on_kw, :on_const, :on_ident].include?(name_t[:kind]) then
warn "expected method name token, . or ::, got #{name_t.inspect}"
skip_method container
return
end
- name_t.name
+ name_t[:text]
end
end
@@ -1416,47 +1455,43 @@ class RDoc::Parser::Ruby < RDoc::Parser
# for a singleton method.
def parse_method_name_singleton container, name_t # :nodoc:
- @scanner.lex_state = :EXPR_FNAME
skip_tkspace
name_t2 = get_tk
- name =
- case name_t
- when TkSELF, TkMOD then
- case name_t2
- # NOTE: work around '[' being consumed early and not being re-tokenized
- # as a TkAREF
- when TkfLBRACK then
- get_tk
- '[]'
- else
- name_t2.name
- end
- when TkCONSTANT then
- name = name_t2.name
-
- container = get_method_container container, name_t
-
- return unless container
-
- name
- when TkIDENTIFIER, TkIVAR, TkGVAR then
- parse_method_dummy container
-
- nil
- when TkTRUE, TkFALSE, TkNIL then
- klass_name = "#{name_t.name.capitalize}Class"
- container = @store.find_class_named klass_name
- container ||= @top_level.add_class RDoc::NormalClass, klass_name
-
- name_t2.name
+ if (:on_kw == name_t[:kind] && 'self' == name_t[:text]) || (:on_op == name_t[:kind] && '%' == name_t[:text]) then
+ # NOTE: work around '[' being consumed early and not being re-tokenized
+ # as a TkAREF
+ if :on_lbracket == name_t2[:kind]
+ get_tk
+ name = '[]'
else
- warn "unexpected method name token #{name_t.inspect}"
- # break
- skip_method container
-
- nil
+ name = name_t2[:text]
end
+ elsif :on_const == name_t[:kind] then
+ name = name_t2[:text]
+
+ container = get_method_container container, name_t
+
+ return unless container
+
+ name
+ elsif :on_ident == name_t[:kind] || :on_ivar == name_t[:kind] || :on_gvar == name_t[:kind] then
+ parse_method_dummy container
+
+ name = nil
+ elsif (:on_kw == name_t[:kind]) && ('true' == name_t[:text] || 'false' == name_t[:text] || 'nil' == name_t[:text]) then
+ klass_name = "#{name_t[:text].capitalize}Class"
+ container = @store.find_class_named klass_name
+ container ||= @top_level.add_class RDoc::NormalClass, klass_name
+
+ name = name_t2[:text]
+ else
+ warn "unexpected method name token #{name_t.inspect}"
+ # break
+ skip_method container
+
+ name = nil
+ end
return name, container
end
@@ -1472,43 +1507,49 @@ class RDoc::Parser::Ruby < RDoc::Parser
return '' unless end_token
nest = 0
+ continue = false
- loop do
- case tk
- when TkSEMICOLON then
+ while tk != nil do
+ case tk[:kind]
+ when :on_semicolon then
break if nest == 0
- when TkLBRACE, TkfLBRACE then
+ when :on_lbrace then
nest += 1
- when TkRBRACE then
+ when :on_rbrace then
nest -= 1
if nest <= 0
# we might have a.each { |i| yield i }
unget_tk(tk) if nest < 0
break
end
- when TkLPAREN, TkfLPAREN then
+ when :on_lparen then
nest += 1
- when end_token then
- if end_token == TkRPAREN
+ when end_token[:kind] then
+ if end_token[:kind] == :on_rparen
nest -= 1
break if nest <= 0
else
- break unless @scanner.continue
+ break
end
- when TkRPAREN then
+ when :on_rparen then
nest -= 1
- when method && method.block_params.nil? && TkCOMMENT then
- unget_tk tk
- read_documentation_modifiers method, modifiers
+ when :on_comment, :on_embdoc then
@read.pop
- when TkCOMMENT then
- @read.pop
- when nil then
- break
+ if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] and
+ (!continue or (RDoc::RipperStateLex::EXPR_LABEL & tk[:state]) != 0) then
+ if method && method.block_params.nil? then
+ unget_tk tk
+ read_documentation_modifiers method, modifiers
+ end
+ break if !continue and nest <= 0
+ end
+ when :on_comma then
+ continue = true
+ when :on_ident then
+ continue = false if continue
end
tk = get_tk
end
- @scanner.first_in_method_statement = true
get_tkread_clean(/\s+/, ' ')
end
@@ -1539,7 +1580,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
def parse_module container, single, tk, comment
container, name_t, = get_class_or_module container
- name = name_t.name
+ name = name_t[:text]
mod = container.add_module RDoc::NormalModule, name
mod.ignore unless container.document_children
@@ -1562,12 +1603,12 @@ class RDoc::Parser::Ruby < RDoc::Parser
skip_tkspace_comment
tk = get_tk
- if TkLPAREN === tk then
+ if :on_lparen == tk[:kind] then
skip_tkspace_comment
tk = get_tk
end
- name = tk.text if TkSTRING === tk
+ name = tk[:text][1..-2] if :on_tstring == tk[:kind]
if name then
@top_level.add_require RDoc::Require.new(name, comment)
@@ -1582,14 +1623,15 @@ class RDoc::Parser::Ruby < RDoc::Parser
def parse_rescue
skip_tkspace false
+ nest = 0
while tk = get_tk
- case tk
- when TkNL, TkSEMICOLON then
+ case tk[:kind]
+ when :on_nl, :on_semicolon, :on_comment then
break
- when TkCOMMA then
+ when :on_comma then
skip_tkspace false
- get_tk if TkNL === peek_tk
+ get_tk if :on_nl == peek_tk[:kind]
end
skip_tkspace false
@@ -1613,14 +1655,32 @@ class RDoc::Parser::Ruby < RDoc::Parser
keep_comment = false
try_parse_comment = false
- non_comment_seen = true unless TkCOMMENT === tk
+ non_comment_seen = true unless (:on_comment == tk[:kind] or :on_embdoc == tk[:kind])
- case tk
- when TkNL then
- skip_tkspace
- tk = get_tk
+ case tk[:kind]
+ when :on_nl, :on_ignored_nl, :on_comment, :on_embdoc then
+ if :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]
+ skip_tkspace
+ tk = get_tk
+ else
+ past_tokens = @read.size > 1 ? @read[0..-2] : []
+ nl_position = 0
+ past_tokens.reverse.each_with_index do |read_tk, i|
+ if read_tk =~ /^\n$/ then
+ nl_position = (past_tokens.size - 1) - i
+ break
+ elsif read_tk =~ /^#.*\n$/ then
+ nl_position = ((past_tokens.size - 1) - i) + 1
+ break
+ end
+ end
+ comment_only_line = past_tokens[nl_position..-1].all?{ |c| c =~ /^\s+$/ }
+ unless comment_only_line then
+ tk = get_tk
+ end
+ end
- if TkCOMMENT === tk then
+ if tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then
if non_comment_seen then
# Look for RDoc in a comment about to be thrown away
non_comment_seen = parse_comment container, tk, comment unless
@@ -1630,15 +1690,14 @@ class RDoc::Parser::Ruby < RDoc::Parser
comment.force_encoding @encoding if @encoding
end
- while TkCOMMENT === tk do
- comment << tk.text << "\n"
+ while tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) do
+ comment << tk[:text]
+ comment << "\n" unless "\n" == tk[:text].chars.to_a.last
- tk = get_tk
-
- if TkNL === tk then
+ if tk[:text].size > 1 && "\n" == tk[:text].chars.to_a.last then
skip_tkspace false # leading spaces
- tk = get_tk
end
+ tk = get_tk
end
comment = new_comment comment
@@ -1661,59 +1720,76 @@ class RDoc::Parser::Ruby < RDoc::Parser
keep_comment = true
container.current_line_visibility = nil
- when TkCLASS then
- parse_class container, single, tk, comment
+ when :on_kw then
+ case tk[:text]
+ when 'class' then
+ parse_class container, single, tk, comment
- when TkMODULE then
- parse_module container, single, tk, comment
+ when 'module' then
+ parse_module container, single, tk, comment
- when TkDEF then
- parse_method container, single, tk, comment
+ when 'def' then
+ parse_method container, single, tk, comment
- when TkCONSTANT then
+ when 'alias' then
+ parse_alias container, single, tk, comment unless current_method
+
+ when 'yield' then
+ if current_method.nil? then
+ warn "Warning: yield outside of method" if container.document_self
+ else
+ parse_yield container, single, tk, current_method
+ end
+
+ when 'until', 'while' then
+ if (RDoc::RipperStateLex::EXPR_LABEL & tk[:state]) == 0
+ nest += 1
+ skip_optional_do_after_expression
+ end
+
+ # Until and While can have a 'do', which shouldn't increase the nesting.
+ # We can't solve the general case, but we can handle most occurrences by
+ # ignoring a do at the end of a line.
+
+ # 'for' is trickier
+ when 'for' then
+ nest += 1
+ skip_for_variable
+ skip_optional_do_after_expression
+
+ when 'case', 'do', 'if', 'unless', 'begin' then
+ if (RDoc::RipperStateLex::EXPR_LABEL & tk[:state]) == 0
+ nest += 1
+ end
+
+ when 'super' then
+ current_method.calls_super = true if current_method
+
+ when 'rescue' then
+ parse_rescue
+
+ when 'end' then
+ nest -= 1
+ if nest == 0 then
+ container.ongoing_visibility = save_visibility
+
+ parse_comment container, tk, comment unless comment.empty?
+
+ return
+ end
+ end
+
+ when :on_const then
unless parse_constant container, tk, comment, current_method then
try_parse_comment = true
end
- when TkALIAS then
- parse_alias container, single, tk, comment unless current_method
-
- when TkYIELD then
- if current_method.nil? then
- warn "Warning: yield outside of method" if container.document_self
- else
- parse_yield container, single, tk, current_method
- end
-
- # Until and While can have a 'do', which shouldn't increase the nesting.
- # We can't solve the general case, but we can handle most occurrences by
- # ignoring a do at the end of a line.
-
- when TkUNTIL, TkWHILE then
- nest += 1
- skip_optional_do_after_expression
-
- # 'for' is trickier
- when TkFOR then
- nest += 1
- skip_for_variable
- skip_optional_do_after_expression
-
- when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN then
- nest += 1
-
- when TkSUPER then
- current_method.calls_super = true if current_method
-
- when TkRESCUE then
- parse_rescue
-
- when TkIDENTIFIER then
+ when :on_ident then
if nest == 1 and current_method.nil? then
keep_comment = parse_identifier container, single, tk, comment
end
- case tk.name
+ case tk[:text]
when "require" then
parse_require container, comment
when "include" then
@@ -1722,15 +1798,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
parse_extend_or_include RDoc::Extend, container, comment
end
- when TkEND then
- nest -= 1
- if nest == 0 then
- container.ongoing_visibility = save_visibility
-
- parse_comment container, tk, comment unless comment.empty?
-
- return
- end
else
try_parse_comment = nest == 1
end
@@ -1762,8 +1829,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
def parse_symbol_arg(no = nil)
skip_tkspace_comment
- case tk = get_tk
- when TkLPAREN
+ tk = get_tk
+ if tk[:kind] == :on_lparen
parse_symbol_arg_paren no
else
parse_symbol_arg_space no, tk
@@ -1785,10 +1852,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
end
skip_tkspace_comment
- case tk2 = get_tk
- when TkRPAREN
+ case (tk2 = get_tk)[:kind]
+ when :on_rparen
break
- when TkCOMMA
+ when :on_comma
else
warn("unexpected token: '#{tk2.inspect}'") if $DEBUG_RDOC
break
@@ -1815,7 +1882,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
skip_tkspace false
tk1 = get_tk
- unless TkCOMMA === tk1 then
+ if tk1.nil? || :on_comma != tk1[:kind] then
unget_tk tk1
break
end
@@ -1834,12 +1901,12 @@ class RDoc::Parser::Ruby < RDoc::Parser
# Returns symbol text from the next token
def parse_symbol_in_arg
- case tk = get_tk
- when TkSYMBOL
- tk.text.sub(/^:/, '')
- when TkSTRING
- eval @read[-1]
- when TkDSTRING, TkIDENTIFIER then
+ tk = get_tk
+ if :on_symbol == tk[:kind] then
+ tk[:text].sub(/^:/, '')
+ elsif :on_tstring == tk[:kind] then
+ tk[:text][1..-2]
+ elsif :on_dstring == tk[:kind] or :on_ident == tk[:kind] then
nil # ignore
else
warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG_RDOC
@@ -1873,21 +1940,37 @@ class RDoc::Parser::Ruby < RDoc::Parser
skip_tkspace_comment false
- case peek_tk
- # Ryan Davis suggested the extension to ignore modifiers, because he
- # often writes
- #
- # protected unless $TESTING
- #
- when TkNL, TkUNLESS_MOD, TkIF_MOD, TkSEMICOLON then
+ ptk = peek_tk
+ # Ryan Davis suggested the extension to ignore modifiers, because he
+ # often writes
+ #
+ # protected unless $TESTING
+ #
+ if [:on_nl, :on_semicolon].include?(ptk[:kind]) || (:on_kw == ptk[:kind] && (['if', 'unless'].include?(ptk[:text]))) then
container.ongoing_visibility = vis
- when TkDEF
+ elsif :on_kw == ptk[:kind] && 'def' == ptk[:text]
container.current_line_visibility = vis
else
update_visibility container, vis_type, vis, singleton
end
end
+ ##
+ # Parses a Module#private_constant or Module#public_constant call from +tk+.
+
+ def parse_constant_visibility(container, single, tk)
+ args = parse_symbol_arg
+ case tk[:text]
+ when 'private_constant'
+ vis = :private
+ when 'public_constant'
+ vis = :public
+ else
+ raise RDoc::Error, 'Unreachable'
+ end
+ container.set_constant_visibility_for args, vis
+ end
+
##
# Determines the block parameter for +context+
@@ -1895,7 +1978,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
return if method.block_params
get_tkread
- @scanner.continue = false
method.block_params = parse_method_or_yield_parameters
end
@@ -1919,11 +2001,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
while tk = get_tk do
tokens << tk
- case tk
- when TkNL, TkDEF then
+ if :on_nl == tk[:kind] or (:on_kw == tk[:kind] && 'def' == tk[:text]) then
return
- when TkCOMMENT then
- return unless tk.text =~ /\s*:?([\w-]+):\s*(.*)/
+ elsif :on_comment == tk[:kind] or :on_embdoc == tk[:kind] then
+ return unless tk[:text] =~ /\s*:?([\w-]+):\s*(.*)/
directive = $1.downcase
@@ -1933,7 +2014,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
end
end
ensure
- unless tokens.length == 1 and TkCOMMENT === tokens.first then
+ unless tokens.length == 1 and (:on_comment == tokens.first[:kind] or :on_embdoc == tokens.first[:kind]) then
tokens.reverse_each do |token|
unget_tk token
end
@@ -1947,6 +2028,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
# See also RDoc::Markup::PreProcess#handle_directive
def read_documentation_modifiers context, allowed
+ skip_tkspace(false)
directive, value = read_directive allowed
return unless directive
@@ -1995,27 +2077,25 @@ class RDoc::Parser::Ruby < RDoc::Parser
rescue StandardError => e
bytes = ''
- 20.times do @scanner.ungetc end
- count = 0
- 60.times do |i|
- count = i
- byte = @scanner.getc
- break unless byte
- bytes << byte
+ if @scanner_point >= @scanner.size
+ now_line_no = @scanner[@scanner.size - 1][:line_no]
+ else
+ now_line_no = peek_tk[:line_no]
end
- count -= 20
- count.times do @scanner.ungetc end
$stderr.puts <<-EOF
-#{self.class} failure around line #{@scanner.line_no} of
+#{self.class} failure around line #{now_line_no} of
#{@file_name}
EOF
unless bytes.empty? then
$stderr.puts
- $stderr.puts bytes.inspect
+ now_line_no = peek_tk[:line_no]
+ start_index = @scanner.find_index { |tk| tk[:line_no] == now_line_no }
+ end_index = @scanner.find_index { |tk| tk[:line_no] == now_line_no + 1 } - 1
+ $stderr.puts @scanner[start_index..end_index].join
end
raise e
@@ -2034,31 +2114,36 @@ class RDoc::Parser::Ruby < RDoc::Parser
b_nest = 0
nest = 0
- @scanner.continue = false
loop do
- case tk
- when TkSEMICOLON, TkNL then
+ break unless tk
+ case tk[:kind]
+ when :on_semicolon, :on_nl, :on_ignored_nl then
break if b_nest.zero?
- when TkLPAREN, TkfLPAREN then
+ when :on_lparen then
nest += 1
- when TkRPAREN then
+ when :on_rparen then
nest -= 1
- when TkBEGIN then
- b_nest += 1
- when TkEND then
- b_nest -= 1
- when TkDO
- break if nest.zero?
- when nil then
- break
+ when :on_kw then
+ case tk[:text]
+ when 'begin'
+ b_nest += 1
+ when 'end'
+ b_nest -= 1
+ when 'do'
+ break if nest.zero?
+ end
+ when :on_comment, :on_embdoc then
+ if b_nest.zero? and "\n" == tk[:text][-1] then
+ break
+ end
end
tk = get_tk
end
skip_tkspace false
- get_tk if TkDO === peek_tk
+ get_tk if peek_tk && :on_kw == peek_tk[:kind] && 'do' == peek_tk[:text]
end
##
@@ -2069,7 +2154,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
get_tk
skip_tkspace false
tk = get_tk
- unget_tk(tk) unless TkIN === tk
+ unget_tk(tk) unless :on_kw == tk[:kind] and 'in' == tk[:text]
end
##
@@ -2087,7 +2172,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
def skip_tkspace_comment(skip_nl = true)
loop do
skip_tkspace skip_nl
- return unless TkCOMMENT === peek_tk
+ next_tk = peek_tk
+ return if next_tk.nil? || (:on_comment != next_tk[:kind] and :on_embdoc != next_tk[:kind])
get_tk
end
end
diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb
index e35b420f30..0a566827a5 100644
--- a/lib/rdoc/parser/ruby_tools.rb
+++ b/lib/rdoc/parser/ruby_tools.rb
@@ -5,8 +5,6 @@
module RDoc::Parser::RubyTools
- include RDoc::RubyToken
-
##
# Adds a token listener +obj+, but you should probably use token_listener
@@ -22,16 +20,25 @@ module RDoc::Parser::RubyTools
tk = nil
if @tokens.empty? then
- tk = @scanner.token
- @read.push @scanner.get_readed
- puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
+ if @scanner_point >= @scanner.size
+ return nil
+ else
+ tk = @scanner[@scanner_point]
+ @scanner_point += 1
+ @read.push tk[:text]
+ puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
+ end
else
@read.push @unget_read.shift
tk = @tokens.shift
puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG
end
- tk = nil if TkEND_OF_SCRIPT === tk
+ if tk == nil || :on___end__ == tk[:kind]
+ tk = nil
+ end
+
+ return nil unless tk
# inform any listeners of our shiny new token
@token_listeners.each do |obj|
@@ -102,19 +109,24 @@ module RDoc::Parser::RubyTools
@tokens = []
@unget_read = []
@nest = 0
+ @scanner_point = 0
+ end
+
+ def tk_nl?(tk)
+ :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]
end
##
# Skips whitespace tokens including newlines if +skip_nl+ is true
- def skip_tkspace(skip_nl = true) # HACK dup
+ def skip_tkspace(skip_nl = true)
tokens = []
- while TkSPACE === (tk = get_tk) or (skip_nl and TkNL === tk) do
- tokens.push tk
+ while (tk = get_tk) and (:on_sp == tk[:kind] or (skip_nl and tk_nl?(tk))) do
+ tokens.push(tk)
end
- unget_tk tk
+ unget_tk(tk)
tokens
end
diff --git a/lib/rdoc/rd/block_parser.rb b/lib/rdoc/rd/block_parser.rb
index 3f4941168f..8ac7cad996 100644
--- a/lib/rdoc/rd/block_parser.rb
+++ b/lib/rdoc/rd/block_parser.rb
@@ -677,54 +677,54 @@ Racc_debug_parser = false
# reduce 0 omitted
def _reduce_1(val, _values, result)
- result = RDoc::Markup::Document.new(*val[0])
+ result = RDoc::Markup::Document.new(*val[0])
result
end
def _reduce_2(val, _values, result)
- raise ParseError, "file empty"
+ raise ParseError, "file empty"
result
end
def _reduce_3(val, _values, result)
- result = val[0].concat val[1]
+ result = val[0].concat val[1]
result
end
def _reduce_4(val, _values, result)
- result = val[0]
+ result = val[0]
result
end
def _reduce_5(val, _values, result)
- result = val
+ result = val
result
end
def _reduce_6(val, _values, result)
- result = val
+ result = val
result
end
# reduce 7 omitted
def _reduce_8(val, _values, result)
- result = val
+ result = val
result
end
def _reduce_9(val, _values, result)
- result = val
+ result = val
result
end
def _reduce_10(val, _values, result)
- result = [RDoc::Markup::BlankLine.new]
+ result = [RDoc::Markup::BlankLine.new]
result
end
def _reduce_11(val, _values, result)
- result = val[0].parts
+ result = val[0].parts
result
end
@@ -732,30 +732,30 @@ def _reduce_12(val, _values, result)
# val[0] is like [level, title]
title = @inline_parser.parse(val[0][1])
result = RDoc::Markup::Heading.new(val[0][0], title)
-
+
result
end
def _reduce_13(val, _values, result)
result = RDoc::Markup::Include.new val[0], @include_path
-
+
result
end
def _reduce_14(val, _values, result)
# val[0] is Array of String
result = paragraph val[0]
-
+
result
end
def _reduce_15(val, _values, result)
- result << val[1].rstrip
+ result << val[1].rstrip
result
end
def _reduce_16(val, _values, result)
- result = [val[0].rstrip]
+ result = [val[0].rstrip]
result
end
@@ -766,7 +766,7 @@ def _reduce_17(val, _values, result)
# imform to lexer.
@in_verbatim = false
-
+
result
end
@@ -777,25 +777,25 @@ def _reduce_18(val, _values, result)
# imform to lexer.
@in_verbatim = false
-
+
result
end
def _reduce_19(val, _values, result)
result << val[1]
-
+
result
end
def _reduce_20(val, _values, result)
result.concat val[2]
-
+
result
end
def _reduce_21(val, _values, result)
result << "\n"
-
+
result
end
@@ -803,7 +803,7 @@ def _reduce_22(val, _values, result)
result = val
# inform to lexer.
@in_verbatim = true
-
+
result
end
@@ -817,89 +817,89 @@ end
def _reduce_27(val, _values, result)
result = val[0]
-
+
result
end
def _reduce_28(val, _values, result)
result = val[1]
-
+
result
end
def _reduce_29(val, _values, result)
result = val[1].push(val[2])
-
+
result
end
def _reduce_30(val, _values, result)
- result = val[0] << val[1]
+ result = val[0] << val[1]
result
end
def _reduce_31(val, _values, result)
- result = [val[0]]
+ result = [val[0]]
result
end
def _reduce_32(val, _values, result)
result = RDoc::Markup::List.new :BULLET, *val[0]
-
+
result
end
def _reduce_33(val, _values, result)
- result.push(val[1])
+ result.push(val[1])
result
end
def _reduce_34(val, _values, result)
- result = val
+ result = val
result
end
def _reduce_35(val, _values, result)
result = RDoc::Markup::ListItem.new nil, val[0], *val[1]
-
+
result
end
def _reduce_36(val, _values, result)
result = RDoc::Markup::List.new :NUMBER, *val[0]
-
+
result
end
def _reduce_37(val, _values, result)
- result.push(val[1])
+ result.push(val[1])
result
end
def _reduce_38(val, _values, result)
- result = val
+ result = val
result
end
def _reduce_39(val, _values, result)
result = RDoc::Markup::ListItem.new nil, val[0], *val[1]
-
+
result
end
def _reduce_40(val, _values, result)
result = RDoc::Markup::List.new :NOTE, *val[0]
-
+
result
end
def _reduce_41(val, _values, result)
- result.push(val[1])
+ result.push(val[1])
result
end
def _reduce_42(val, _values, result)
- result = val
+ result = val
result
end
@@ -907,77 +907,77 @@ def _reduce_43(val, _values, result)
term = @inline_parser.parse val[0].strip
result = RDoc::Markup::ListItem.new term, *val[1]
-
+
result
end
def _reduce_44(val, _values, result)
result = RDoc::Markup::List.new :LABEL, *val[0]
-
+
result
end
def _reduce_45(val, _values, result)
- result.push(val[1])
+ result.push(val[1])
result
end
def _reduce_46(val, _values, result)
- result = val
+ result = val
result
end
def _reduce_47(val, _values, result)
result = RDoc::Markup::ListItem.new "#{val[0].strip}", *val[1]
-
+
result
end
def _reduce_48(val, _values, result)
result = [val[1]].concat(val[2])
-
+
result
end
def _reduce_49(val, _values, result)
result = [val[1]]
-
+
result
end
def _reduce_50(val, _values, result)
result = val[2]
-
+
result
end
def _reduce_51(val, _values, result)
result = []
-
+
result
end
def _reduce_52(val, _values, result)
- result.concat val[1]
+ result.concat val[1]
result
end
# reduce 53 omitted
def _reduce_54(val, _values, result)
- result = val
+ result = val
result
end
def _reduce_55(val, _values, result)
- result = val
+ result = val
result
end
# reduce 56 omitted
def _reduce_57(val, _values, result)
- result = []
+ result = []
result
end
@@ -991,58 +991,58 @@ end
def _reduce_62(val, _values, result)
result = paragraph [val[0]].concat(val[1])
-
+
result
end
def _reduce_63(val, _values, result)
result = paragraph [val[0]]
-
+
result
end
def _reduce_64(val, _values, result)
result = paragraph [val[0]].concat(val[1])
-
+
result
end
def _reduce_65(val, _values, result)
result = paragraph [val[0]]
-
+
result
end
def _reduce_66(val, _values, result)
result = [val[0]].concat(val[1])
-
+
result
end
def _reduce_67(val, _values, result)
- result.concat val[1]
+ result.concat val[1]
result
end
def _reduce_68(val, _values, result)
- result = val[1]
+ result = val[1]
result
end
def _reduce_69(val, _values, result)
- result = val
+ result = val
result
end
# reduce 70 omitted
def _reduce_71(val, _values, result)
- result = []
+ result = []
result
end
def _reduce_72(val, _values, result)
- result = []
+ result = []
result
end
diff --git a/lib/rdoc/rd/inline_parser.rb b/lib/rdoc/rd/inline_parser.rb
index cc63ea6f70..4a3f3ed7b5 100644
--- a/lib/rdoc/rd/inline_parser.rb
+++ b/lib/rdoc/rd/inline_parser.rb
@@ -732,12 +732,12 @@ Racc_debug_parser = false
# reduce 1 omitted
def _reduce_2(val, _values, result)
- result.append val[1]
+ result.append val[1]
result
end
def _reduce_3(val, _values, result)
- result = val[0]
+ result = val[0]
result
end
@@ -762,28 +762,28 @@ end
def _reduce_13(val, _values, result)
content = val[1]
result = inline "#{content}", content
-
+
result
end
def _reduce_14(val, _values, result)
content = val[1]
result = inline "#{content}", content
-
+
result
end
def _reduce_15(val, _values, result)
content = val[1]
result = inline "+#{content}+", content
-
+
result
end
def _reduce_16(val, _values, result)
content = val[1]
result = inline "#{content}", content
-
+
result
end
@@ -791,13 +791,13 @@ def _reduce_17(val, _values, result)
label = val[1]
@block_parser.add_label label.reference
result = "#{label}"
-
+
result
end
def _reduce_18(val, _values, result)
result = "{#{val[1]}}[#{val[2].join}]"
-
+
result
end
@@ -805,13 +805,13 @@ def _reduce_19(val, _values, result)
scheme, inline = val[1]
result = "{#{inline}}[#{scheme}#{inline.reference}]"
-
+
result
end
def _reduce_20(val, _values, result)
result = [nil, inline(val[1])]
-
+
result
end
@@ -820,25 +820,25 @@ def _reduce_21(val, _values, result)
'rdoc-label:',
inline("#{val[0].reference}/#{val[1].reference}")
]
-
+
result
end
def _reduce_22(val, _values, result)
result = ['rdoc-label:', val[0].reference]
-
+
result
end
def _reduce_23(val, _values, result)
result = ['rdoc-label:', "#{val[0].reference}/"]
-
+
result
end
def _reduce_24(val, _values, result)
result = [nil, inline(val[1])]
-
+
result
end
@@ -847,92 +847,92 @@ def _reduce_25(val, _values, result)
'rdoc-label:',
inline("#{val[0].reference}/#{val[1].reference}")
]
-
+
result
end
def _reduce_26(val, _values, result)
result = ['rdoc-label:', val[0]]
-
+
result
end
def _reduce_27(val, _values, result)
ref = val[0].reference
result = ['rdoc-label:', inline(ref, "#{ref}/")]
-
+
result
end
# reduce 28 omitted
def _reduce_29(val, _values, result)
- result = val[1]
+ result = val[1]
result
end
def _reduce_30(val, _values, result)
- result = val[1]
+ result = val[1]
result
end
def _reduce_31(val, _values, result)
result = inline val[0]
-
+
result
end
def _reduce_32(val, _values, result)
result = inline "\"#{val[1]}\""
-
+
result
end
def _reduce_33(val, _values, result)
result = inline val[0]
-
+
result
end
def _reduce_34(val, _values, result)
result = inline "\"#{val[1]}\""
-
+
result
end
# reduce 35 omitted
def _reduce_36(val, _values, result)
- result = val[1]
+ result = val[1]
result
end
def _reduce_37(val, _values, result)
- result = inline val[1]
+ result = inline val[1]
result
end
def _reduce_38(val, _values, result)
result = val[0].append val[1]
-
+
result
end
def _reduce_39(val, _values, result)
result = val[0].append val[1]
-
+
result
end
def _reduce_40(val, _values, result)
result = val[0]
-
+
result
end
def _reduce_41(val, _values, result)
result = inline val[0]
-
+
result
end
@@ -940,25 +940,25 @@ end
def _reduce_43(val, _values, result)
result = val[0].append val[1]
-
+
result
end
def _reduce_44(val, _values, result)
result = inline val[0]
-
+
result
end
def _reduce_45(val, _values, result)
result = val[0].append val[1]
-
+
result
end
def _reduce_46(val, _values, result)
result = val[0]
-
+
result
end
@@ -984,24 +984,24 @@ end
def _reduce_57(val, _values, result)
result = val[0]
-
+
result
end
def _reduce_58(val, _values, result)
result = inline val[0]
-
+
result
end
def _reduce_59(val, _values, result)
result = inline val[0]
-
+
result
end
def _reduce_60(val, _values, result)
- result << val[1]
+ result << val[1]
result
end
@@ -1009,7 +1009,7 @@ end
def _reduce_62(val, _values, result)
result << val[1]
-
+
result
end
@@ -1017,7 +1017,7 @@ end
def _reduce_64(val, _values, result)
result << val[1]
-
+
result
end
@@ -1048,7 +1048,7 @@ end
# reduce 77 omitted
def _reduce_78(val, _values, result)
- result << val[1]
+ result << val[1]
result
end
@@ -1099,13 +1099,13 @@ end
def _reduce_101(val, _values, result)
index = @block_parser.add_footnote val[1].rdoc
result = "{*#{index}}[rdoc-label:foottext-#{index}:footmark-#{index}]"
-
+
result
end
def _reduce_102(val, _values, result)
result = inline "#{val[1]}", val[1]
-
+
result
end
@@ -1122,7 +1122,7 @@ end
# reduce 108 omitted
def _reduce_109(val, _values, result)
- result << val[1]
+ result << val[1]
result
end
@@ -1130,24 +1130,24 @@ end
def _reduce_111(val, _values, result)
result = inline val[0]
-
+
result
end
# reduce 112 omitted
def _reduce_113(val, _values, result)
- result = val[1]
+ result = val[1]
result
end
def _reduce_114(val, _values, result)
- result = val[1]
+ result = val[1]
result
end
def _reduce_115(val, _values, result)
- result = val[1]
+ result = val[1]
result
end
@@ -1192,7 +1192,7 @@ end
# reduce 135 omitted
def _reduce_136(val, _values, result)
- result << val[1]
+ result << val[1]
result
end
diff --git a/lib/rdoc/rdoc.gemspec b/lib/rdoc/rdoc.gemspec
index 1822c27366..4c7cd92549 100644
--- a/lib/rdoc/rdoc.gemspec
+++ b/lib/rdoc/rdoc.gemspec
@@ -1,11 +1,10 @@
-# -*- encoding: utf-8 -*-
-$:.unshift File.expand_path("../lib", __FILE__)
-require 'rdoc'
+require_relative "lib/rdoc"
Gem::Specification.new do |s|
s.name = "rdoc"
s.version = RDoc::VERSION
-
+ s.date = "2017-09-12"
+
s.authors = [
"Eric Hodel",
"Dave Thomas",
@@ -28,7 +27,7 @@ RDoc includes the +rdoc+ and +ri+ tools for generating and displaying documentat
s.executables = ["rdoc", "ri"]
s.require_paths = ["lib"]
# for ruby core repository. It was generated by `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
- s.files = [".document", ".gitignore", ".travis.yml", "CONTRIBUTING.rdoc", "CVE-2013-0256.rdoc", "ExampleMarkdown.md", "ExampleRDoc.rdoc", "Gemfile", "History.rdoc", "LEGAL.rdoc", "LICENSE.rdoc", "README.rdoc", "RI.rdoc", "Rakefile", "TODO.rdoc", "bin/console", "bin/setup", "exe/rdoc", "exe/ri", "lib/gauntlet_rdoc.rb", "lib/rdoc.rb", "lib/rdoc/alias.rb", "lib/rdoc/anon_class.rb", "lib/rdoc/any_method.rb", "lib/rdoc/attr.rb", "lib/rdoc/class_module.rb", "lib/rdoc/code_object.rb", "lib/rdoc/code_objects.rb", "lib/rdoc/comment.rb", "lib/rdoc/constant.rb", "lib/rdoc/context.rb", "lib/rdoc/context/section.rb", "lib/rdoc/cross_reference.rb", "lib/rdoc/encoding.rb", "lib/rdoc/erb_partial.rb", "lib/rdoc/erbio.rb", "lib/rdoc/extend.rb", "lib/rdoc/generator.rb", "lib/rdoc/generator/darkfish.rb", "lib/rdoc/generator/json_index.rb", "lib/rdoc/generator/markup.rb", "lib/rdoc/generator/pot.rb", "lib/rdoc/generator/pot/message_extractor.rb", "lib/rdoc/generator/pot/po.rb", "lib/rdoc/generator/pot/po_entry.rb", "lib/rdoc/generator/ri.rb", "lib/rdoc/generator/template/darkfish/.document", "lib/rdoc/generator/template/darkfish/_footer.rhtml", "lib/rdoc/generator/template/darkfish/_head.rhtml","lib/rdoc/generator/template/darkfish/_sidebar_VCS_info.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_classes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_extends.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_in_files.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_includes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_installed.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_methods.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_navigation.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_pages.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_parent.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_search.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_sections.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_table_of_contents.rhtml", "lib/rdoc/generator/template/darkfish/class.rhtml", "lib/rdoc/generator/template/darkfish/css/fonts.css", "lib/rdoc/generator/template/darkfish/css/rdoc.css", "lib/rdoc/generator/template/darkfish/fonts/Lato-Light.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-LightItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-Regular.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-RegularItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Bold.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Regular.ttf", "lib/rdoc/generator/template/darkfish/images/add.png", "lib/rdoc/generator/template/darkfish/images/arrow_up.png", "lib/rdoc/generator/template/darkfish/images/brick.png", "lib/rdoc/generator/template/darkfish/images/brick_link.png", "lib/rdoc/generator/template/darkfish/images/bug.png", "lib/rdoc/generator/template/darkfish/images/bullet_black.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_minus.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_plus.png", "lib/rdoc/generator/template/darkfish/images/date.png", "lib/rdoc/generator/template/darkfish/images/delete.png", "lib/rdoc/generator/template/darkfish/images/find.png", "lib/rdoc/generator/template/darkfish/images/loadingAnimation.gif", "lib/rdoc/generator/template/darkfish/images/macFFBgHack.png", "lib/rdoc/generator/template/darkfish/images/package.png", "lib/rdoc/generator/template/darkfish/images/page_green.png", "lib/rdoc/generator/template/darkfish/images/page_white_text.png", "lib/rdoc/generator/template/darkfish/images/page_white_width.png", "lib/rdoc/generator/template/darkfish/images/plugin.png", "lib/rdoc/generator/template/darkfish/images/ruby.png", "lib/rdoc/generator/template/darkfish/images/tag_blue.png", "lib/rdoc/generator/template/darkfish/images/tag_green.png", "lib/rdoc/generator/template/darkfish/images/transparent.png", "lib/rdoc/generator/template/darkfish/images/wrench.png", "lib/rdoc/generator/template/darkfish/images/wrench_orange.png", "lib/rdoc/generator/template/darkfish/images/zoom.png", "lib/rdoc/generator/template/darkfish/index.rhtml", "lib/rdoc/generator/template/darkfish/js/darkfish.js", "lib/rdoc/generator/template/darkfish/js/jquery.js", "lib/rdoc/generator/template/darkfish/js/search.js", "lib/rdoc/generator/template/darkfish/page.rhtml", "lib/rdoc/generator/template/darkfish/servlet_not_found.rhtml", "lib/rdoc/generator/template/darkfish/servlet_root.rhtml", "lib/rdoc/generator/template/darkfish/table_of_contents.rhtml", "lib/rdoc/generator/template/json_index/.document", "lib/rdoc/generator/template/json_index/js/navigation.js", "lib/rdoc/generator/template/json_index/js/searcher.js", "lib/rdoc/ghost_method.rb", "lib/rdoc/i18n.rb", "lib/rdoc/i18n/locale.rb", "lib/rdoc/i18n/text.rb", "lib/rdoc/include.rb", "lib/rdoc/known_classes.rb", "lib/rdoc/markdown.kpeg", "lib/rdoc/markdown/entities.rb", "lib/rdoc/markdown/literals.kpeg", "lib/rdoc/markdown/literals.rb", "lib/rdoc/markup.rb", "lib/rdoc/markup/attr_changer.rb", "lib/rdoc/markup/attr_span.rb", "lib/rdoc/markup/attribute_manager.rb", "lib/rdoc/markup/attributes.rb", "lib/rdoc/markup/blank_line.rb", "lib/rdoc/markup/block_quote.rb", "lib/rdoc/markup/document.rb", "lib/rdoc/markup/formatter.rb", "lib/rdoc/markup/formatter_test_case.rb", "lib/rdoc/markup/hard_break.rb", "lib/rdoc/markup/heading.rb", "lib/rdoc/markup/include.rb", "lib/rdoc/markup/indented_paragraph.rb", "lib/rdoc/markup/inline.rb", "lib/rdoc/markup/list.rb", "lib/rdoc/markup/list_item.rb", "lib/rdoc/markup/paragraph.rb", "lib/rdoc/markup/parser.rb", "lib/rdoc/markup/pre_process.rb", "lib/rdoc/markup/raw.rb", "lib/rdoc/markup/rule.rb", "lib/rdoc/markup/special.rb", "lib/rdoc/markup/text_formatter_test_case.rb", "lib/rdoc/markup/to_ansi.rb", "lib/rdoc/markup/to_bs.rb", "lib/rdoc/markup/to_html.rb", "lib/rdoc/markup/to_html_crossref.rb", "lib/rdoc/markup/to_html_snippet.rb", "lib/rdoc/markup/to_joined_paragraph.rb", "lib/rdoc/markup/to_label.rb", "lib/rdoc/markup/to_markdown.rb", "lib/rdoc/markup/to_rdoc.rb", "lib/rdoc/markup/to_table_of_contents.rb", "lib/rdoc/markup/to_test.rb", "lib/rdoc/markup/to_tt_only.rb", "lib/rdoc/markup/verbatim.rb", "lib/rdoc/meta_method.rb", "lib/rdoc/method_attr.rb", "lib/rdoc/mixin.rb", "lib/rdoc/normal_class.rb", "lib/rdoc/normal_module.rb", "lib/rdoc/options.rb", "lib/rdoc/parser.rb", "lib/rdoc/parser/c.rb", "lib/rdoc/parser/changelog.rb", "lib/rdoc/parser/markdown.rb", "lib/rdoc/parser/rd.rb", "lib/rdoc/parser/ruby.rb", "lib/rdoc/parser/ruby_tools.rb", "lib/rdoc/parser/simple.rb", "lib/rdoc/parser/text.rb", "lib/rdoc/rd.rb", "lib/rdoc/rd/block_parser.ry", "lib/rdoc/rd/inline.rb", "lib/rdoc/rd/inline_parser.ry", "lib/rdoc/rdoc.rb", "lib/rdoc/require.rb", "lib/rdoc/ri.rb", "lib/rdoc/ri/driver.rb", "lib/rdoc/ri/formatter.rb", "lib/rdoc/ri/paths.rb", "lib/rdoc/ri/store.rb", "lib/rdoc/ri/task.rb", "lib/rdoc/ruby_lex.rb", "lib/rdoc/ruby_token.rb", "lib/rdoc/rubygems_hook.rb", "lib/rdoc/servlet.rb", "lib/rdoc/single_class.rb", "lib/rdoc/stats.rb", "lib/rdoc/stats/normal.rb", "lib/rdoc/stats/quiet.rb", "lib/rdoc/stats/verbose.rb", "lib/rdoc/store.rb", "lib/rdoc/task.rb", "lib/rdoc/test_case.rb", "lib/rdoc/text.rb", "lib/rdoc/token_stream.rb", "lib/rdoc/tom_doc.rb", "lib/rdoc/top_level.rb", "rdoc.gemspec"]
+ s.files = [".document", ".gitignore", ".travis.yml", "CONTRIBUTING.rdoc", "CVE-2013-0256.rdoc", "ExampleMarkdown.md", "ExampleRDoc.rdoc", "Gemfile", "History.rdoc", "LEGAL.rdoc", "LICENSE.rdoc", "README.rdoc", "RI.rdoc", "Rakefile", "TODO.rdoc", "appveyor.yml", "bin/console", "bin/setup", "exe/rdoc", "exe/ri", "lib/rdoc.rb", "lib/rdoc/alias.rb", "lib/rdoc/anon_class.rb", "lib/rdoc/any_method.rb", "lib/rdoc/attr.rb", "lib/rdoc/class_module.rb", "lib/rdoc/code_object.rb", "lib/rdoc/code_objects.rb", "lib/rdoc/comment.rb", "lib/rdoc/constant.rb", "lib/rdoc/context.rb", "lib/rdoc/context/section.rb", "lib/rdoc/cross_reference.rb", "lib/rdoc/encoding.rb", "lib/rdoc/erb_partial.rb", "lib/rdoc/erbio.rb", "lib/rdoc/extend.rb", "lib/rdoc/generator.rb", "lib/rdoc/generator/darkfish.rb", "lib/rdoc/generator/json_index.rb", "lib/rdoc/generator/markup.rb", "lib/rdoc/generator/pot.rb", "lib/rdoc/generator/pot/message_extractor.rb", "lib/rdoc/generator/pot/po.rb", "lib/rdoc/generator/pot/po_entry.rb", "lib/rdoc/generator/ri.rb", "lib/rdoc/generator/template/darkfish/.document", "lib/rdoc/generator/template/darkfish/_footer.rhtml", "lib/rdoc/generator/template/darkfish/_head.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_VCS_info.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_classes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_extends.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_in_files.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_includes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_installed.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_methods.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_navigation.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_pages.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_parent.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_search.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_sections.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_table_of_contents.rhtml", "lib/rdoc/generator/template/darkfish/class.rhtml", "lib/rdoc/generator/template/darkfish/css/fonts.css", "lib/rdoc/generator/template/darkfish/css/rdoc.css", "lib/rdoc/generator/template/darkfish/fonts/Lato-Light.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-LightItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-Regular.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-RegularItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Bold.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Regular.ttf", "lib/rdoc/generator/template/darkfish/images/add.png", "lib/rdoc/generator/template/darkfish/images/arrow_up.png", "lib/rdoc/generator/template/darkfish/images/brick.png", "lib/rdoc/generator/template/darkfish/images/brick_link.png", "lib/rdoc/generator/template/darkfish/images/bug.png", "lib/rdoc/generator/template/darkfish/images/bullet_black.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_minus.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_plus.png", "lib/rdoc/generator/template/darkfish/images/date.png", "lib/rdoc/generator/template/darkfish/images/delete.png", "lib/rdoc/generator/template/darkfish/images/find.png", "lib/rdoc/generator/template/darkfish/images/loadingAnimation.gif", "lib/rdoc/generator/template/darkfish/images/macFFBgHack.png", "lib/rdoc/generator/template/darkfish/images/package.png", "lib/rdoc/generator/template/darkfish/images/page_green.png", "lib/rdoc/generator/template/darkfish/images/page_white_text.png", "lib/rdoc/generator/template/darkfish/images/page_white_width.png", "lib/rdoc/generator/template/darkfish/images/plugin.png", "lib/rdoc/generator/template/darkfish/images/ruby.png", "lib/rdoc/generator/template/darkfish/images/tag_blue.png", "lib/rdoc/generator/template/darkfish/images/tag_green.png", "lib/rdoc/generator/template/darkfish/images/transparent.png", "lib/rdoc/generator/template/darkfish/images/wrench.png", "lib/rdoc/generator/template/darkfish/images/wrench_orange.png", "lib/rdoc/generator/template/darkfish/images/zoom.png", "lib/rdoc/generator/template/darkfish/index.rhtml", "lib/rdoc/generator/template/darkfish/js/darkfish.js", "lib/rdoc/generator/template/darkfish/js/jquery.js", "lib/rdoc/generator/template/darkfish/js/search.js", "lib/rdoc/generator/template/darkfish/page.rhtml", "lib/rdoc/generator/template/darkfish/servlet_not_found.rhtml", "lib/rdoc/generator/template/darkfish/servlet_root.rhtml", "lib/rdoc/generator/template/darkfish/table_of_contents.rhtml", "lib/rdoc/generator/template/json_index/.document", "lib/rdoc/generator/template/json_index/js/navigation.js", "lib/rdoc/generator/template/json_index/js/searcher.js", "lib/rdoc/ghost_method.rb", "lib/rdoc/i18n.rb", "lib/rdoc/i18n/locale.rb", "lib/rdoc/i18n/text.rb", "lib/rdoc/include.rb", "lib/rdoc/known_classes.rb", "lib/rdoc/markdown.kpeg", "lib/rdoc/markdown/entities.rb", "lib/rdoc/markdown/literals.kpeg", "lib/rdoc/markdown/literals.rb", "lib/rdoc/markup.rb", "lib/rdoc/markup/attr_changer.rb", "lib/rdoc/markup/attr_span.rb", "lib/rdoc/markup/attribute_manager.rb", "lib/rdoc/markup/attributes.rb", "lib/rdoc/markup/blank_line.rb", "lib/rdoc/markup/block_quote.rb", "lib/rdoc/markup/document.rb", "lib/rdoc/markup/formatter.rb", "lib/rdoc/markup/formatter_test_case.rb", "lib/rdoc/markup/hard_break.rb", "lib/rdoc/markup/heading.rb", "lib/rdoc/markup/include.rb", "lib/rdoc/markup/indented_paragraph.rb", "lib/rdoc/markup/inline.rb", "lib/rdoc/markup/list.rb", "lib/rdoc/markup/list_item.rb", "lib/rdoc/markup/paragraph.rb", "lib/rdoc/markup/parser.rb", "lib/rdoc/markup/pre_process.rb", "lib/rdoc/markup/raw.rb", "lib/rdoc/markup/rule.rb", "lib/rdoc/markup/special.rb", "lib/rdoc/markup/text_formatter_test_case.rb", "lib/rdoc/markup/to_ansi.rb", "lib/rdoc/markup/to_bs.rb", "lib/rdoc/markup/to_html.rb", "lib/rdoc/markup/to_html_crossref.rb", "lib/rdoc/markup/to_html_snippet.rb", "lib/rdoc/markup/to_joined_paragraph.rb", "lib/rdoc/markup/to_label.rb", "lib/rdoc/markup/to_markdown.rb", "lib/rdoc/markup/to_rdoc.rb", "lib/rdoc/markup/to_table_of_contents.rb", "lib/rdoc/markup/to_test.rb", "lib/rdoc/markup/to_tt_only.rb", "lib/rdoc/markup/verbatim.rb", "lib/rdoc/meta_method.rb", "lib/rdoc/method_attr.rb", "lib/rdoc/mixin.rb", "lib/rdoc/normal_class.rb", "lib/rdoc/normal_module.rb", "lib/rdoc/options.rb", "lib/rdoc/parser.rb", "lib/rdoc/parser/c.rb", "lib/rdoc/parser/changelog.rb", "lib/rdoc/parser/markdown.rb", "lib/rdoc/parser/rd.rb", "lib/rdoc/parser/ripper_state_lex.rb", "lib/rdoc/parser/ruby.rb", "lib/rdoc/parser/ruby_tools.rb", "lib/rdoc/parser/simple.rb", "lib/rdoc/parser/text.rb", "lib/rdoc/rd.rb", "lib/rdoc/rd/block_parser.ry", "lib/rdoc/rd/inline.rb", "lib/rdoc/rd/inline_parser.ry", "lib/rdoc/rdoc.rb", "lib/rdoc/require.rb", "lib/rdoc/ri.rb", "lib/rdoc/ri/driver.rb", "lib/rdoc/ri/formatter.rb", "lib/rdoc/ri/paths.rb", "lib/rdoc/ri/store.rb", "lib/rdoc/ri/task.rb", "lib/rdoc/ruby_token.rb", "lib/rdoc/rubygems_hook.rb", "lib/rdoc/servlet.rb", "lib/rdoc/single_class.rb", "lib/rdoc/stats.rb", "lib/rdoc/stats/normal.rb", "lib/rdoc/stats/quiet.rb", "lib/rdoc/stats/verbose.rb", "lib/rdoc/store.rb", "lib/rdoc/task.rb", "lib/rdoc/test_case.rb", "lib/rdoc/text.rb", "lib/rdoc/token_stream.rb", "lib/rdoc/tom_doc.rb", "lib/rdoc/top_level.rb", "rdoc.gemspec"]
# files from .gitignore
s.files << "lib/rdoc/rd/block_parser.rb" << "lib/rdoc/rd/inline_parser.rb" << "lib/rdoc/markdown.rb"
@@ -54,4 +53,5 @@ RDoc includes the +rdoc+ and +ri+ tools for generating and displaying documentat
s.add_development_dependency("racc", "> 1.4.10")
s.add_development_dependency("kpeg")
s.add_development_dependency("minitest", "~> 4")
+ s.add_development_dependency("json")
end
diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb
deleted file mode 100644
index e76fdf0414..0000000000
--- a/lib/rdoc/ruby_lex.rb
+++ /dev/null
@@ -1,1521 +0,0 @@
-# coding: US-ASCII
-# frozen_string_literal: false
-
-#--
-# irb/ruby-lex.rb - ruby lexcal analyzer
-# $Release Version: 0.9.5$
-# $Revision: 17979 $
-# $Date: 2008-07-09 10:17:05 -0700 (Wed, 09 Jul 2008) $
-# by Keiju ISHITSUKA(keiju@ruby-lang.org)
-#
-#++
-
-require "e2mmap"
-require "irb/slex"
-require "stringio"
-
-##
-# Ruby lexer adapted from irb.
-#
-# The internals are not documented because they are scary.
-
-class RDoc::RubyLex
-
- ##
- # Raised upon invalid input
-
- class Error < RDoc::Error
- end
-
- # :stopdoc:
-
- extend Exception2MessageMapper
-
- def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
- def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
- def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
- def_exception(:TkReading2TokenDuplicateError,
- "key duplicate(token_n='%s', key='%s')")
- def_exception(:SyntaxError, "%s")
-
- def_exception(:TerminateLineInput, "Terminate Line Input")
-
- include RDoc::RubyToken
- include IRB
-
- attr_accessor :continue
- attr_accessor :lex_state
- attr_accessor :first_in_method_statement
- attr_reader :reader
-
- class << self
- attr_accessor :debug_level
- end
-
- def self.debug?
- @debug_level > 0
- end
-
- self.debug_level = 0
-
- # :startdoc:
-
- ##
- # Returns an Array of +ruby+ tokens. See ::new for a description of
- # +options+.
-
- def self.tokenize ruby, options
- tokens = []
-
- scanner = RDoc::RubyLex.new ruby, options
- scanner.exception_on_syntax_error = true
-
- while token = scanner.token do
- tokens << token
- end
-
- tokens
- end
-
- ##
- # Creates a new lexer for +content+. +options+ is an RDoc::Options, only
- # +tab_width is used.
-
- def initialize(content, options)
- lex_init
-
- if /\t/ =~ content then
- tab_width = options.tab_width
- content = content.split(/\n/).map do |line|
- 1 while line.gsub!(/\t+/) {
- ' ' * (tab_width*$&.length - $`.length % tab_width)
- } && $~
- line
- end.join("\n")
- end
-
- content << "\n" unless content[-1, 1] == "\n"
-
- set_input StringIO.new content
-
- @base_char_no = 0
- @char_no = 0
- @exp_line_no = @line_no = 1
- @here_readed = []
- @readed = []
- @current_readed = @readed
- @rests = []
- @seek = 0
-
- @heredoc_queue = []
-
- @indent = 0
- @indent_stack = []
- @lex_state = :EXPR_BEG
- @space_seen = false
- @escaped_nl = false
- @first_in_method_statement = false
- @after_question = false
-
- @continue = false
- @line = ""
-
- @skip_space = false
- @readed_auto_clean_up = false
- @exception_on_syntax_error = true
-
- @prompt = nil
- @prev_seek = nil
- @ltype = nil
- end
-
- # :stopdoc:
-
- def inspect # :nodoc:
- "#<%s:0x%x pos %d lex_state %p space_seen %p>" % [
- self.class, object_id,
- @io.pos, @lex_state, @space_seen,
- ]
- end
-
- attr_accessor :skip_space
- attr_accessor :readed_auto_clean_up
- attr_accessor :exception_on_syntax_error
-
- attr_reader :seek
- attr_reader :char_no
- attr_reader :line_no
- attr_reader :indent
-
- # io functions
- def set_input(io, p = nil, &block)
- @io = io
- if p.respond_to?(:call)
- @input = p
- elsif block_given?
- @input = block
- else
- @input = Proc.new{@io.gets}
- end
- end
-
- def get_readed
- if idx = @readed.rindex("\n")
- @base_char_no = @readed.size - (idx + 1)
- else
- @base_char_no += @readed.size
- end
-
- readed = @readed.join("")
- @readed.clear
- readed
- end
-
- def getc
- while @rests.empty?
- # return nil unless buf_input
- @rests.push nil unless buf_input
- end
- c = @rests.shift
- @current_readed.push c
- @seek += 1
- if c == "\n".freeze
- @line_no += 1
- @char_no = 0
- else
- @char_no += 1
- end
-
- c
- end
-
- def gets
- l = ""
- while c = getc
- l.concat(c)
- break if c == "\n"
- end
- return nil if l == "" and c.nil?
- l
- end
-
- def eof?
- @io.eof?
- end
-
- def getc_of_rests
- if @rests.empty?
- nil
- else
- getc
- end
- end
-
- def ungetc(c = nil)
- if @here_readed.empty?
- c2 = @readed.pop
- else
- c2 = @here_readed.pop
- end
- c = c2 unless c
- @rests.unshift c #c =
- @seek -= 1
- if c == "\n"
- @line_no -= 1
- if idx = @readed.rindex("\n")
- @char_no = idx + 1
- else
- @char_no = @base_char_no + @readed.size
- end
- else
- @char_no -= 1
- end
- end
-
- def peek_equal?(str)
- chrs = str.split(//)
- until @rests.size >= chrs.size
- return false unless buf_input
- end
- @rests[0, chrs.size] == chrs
- end
-
- def peek_match?(regexp)
- while @rests.empty?
- return false unless buf_input
- end
- regexp =~ @rests.join("")
- end
-
- def peek(i = 0)
- while @rests.size <= i
- return nil unless buf_input
- end
- @rests[i]
- end
-
- def buf_input
- prompt
- line = @input.call
- return nil unless line
- @rests.concat line.split(//)
- true
- end
- private :buf_input
-
- def set_prompt(p = nil, &block)
- p = block if block_given?
- if p.respond_to?(:call)
- @prompt = p
- else
- @prompt = Proc.new{print p}
- end
- end
-
- def prompt
- if @prompt
- @prompt.call(@ltype, @indent, @continue, @line_no)
- end
- end
-
- def initialize_input
- @ltype = nil
- @quoted = nil
- @indent = 0
- @indent_stack = []
- @lex_state = :EXPR_BEG
- @space_seen = false
- @current_readed = @readed
-
- @continue = false
- prompt
-
- @line = ""
- @exp_line_no = @line_no
- end
-
- def each_top_level_statement
- initialize_input
- catch(:TERM_INPUT) do
- loop do
- begin
- @continue = false
- prompt
- unless l = lex
- throw :TERM_INPUT if @line == ''
- else
- #p l
- @line.concat l
- if @ltype or @continue or @indent > 0
- next
- end
- end
- if @line != "\n"
- yield @line, @exp_line_no
- end
- break unless l
- @line = ''
- @exp_line_no = @line_no
-
- @indent = 0
- @indent_stack = []
- prompt
- rescue TerminateLineInput
- initialize_input
- prompt
- get_readed
- end
- end
- end
- end
-
- def lex
- until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
- !@continue or
- tk.nil?)
- #p tk
- #p @lex_state
- #p self
- end
- line = get_readed
- # print self.inspect
- if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
- nil
- else
- line
- end
- end
-
- def token
- # require "tracer"
- # Tracer.on
- @prev_seek = @seek
- @prev_line_no = @line_no
- @prev_char_no = @char_no
- begin
- begin
- tk = @OP.match(self)
- @space_seen = tk.kind_of?(TkSPACE)
- @first_in_method_statement = false if !@space_seen && @first_in_method_statement
- rescue SyntaxError => e
- raise Error, "syntax error: #{e.message}" if
- @exception_on_syntax_error
-
- tk = TkError.new(@seek, @line_no, @char_no)
- end
- end while @skip_space and tk.kind_of?(TkSPACE)
-
- if @readed_auto_clean_up
- get_readed
- end
-
- if TkSYMBEG === tk then
- tk1 = token
- set_token_position tk.seek, tk.line_no, tk.char_no
-
- case tk1
- when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then
- if tk1.respond_to?(:name) then
- tk = Token(TkSYMBOL, ":" + tk1.name)
- else
- tk = Token(TkSYMBOL, ":" + tk1.text)
- end
- else
- tk = tk1
- end
- elsif (TkPLUS === tk or TkMINUS === tk) and peek(0) =~ /\d/ then
- tk1 = token
- set_token_position tk.seek, tk.line_no, tk.char_no
- tk = Token(tk1.class, tk.text + tk1.text)
- end
- @after_question = false if @after_question and !(TkQUESTION === tk)
-
- # Tracer.off
- tk
- end
-
- ENINDENT_CLAUSE = [
- "case", "class", "def", "do", "for", "if",
- "module", "unless", "until", "while", "begin" #, "when"
- ]
-
- DEINDENT_CLAUSE = ["end" #, "when"
- ]
-
- PERCENT_LTYPE = {
- "q" => "\'",
- "Q" => "\"",
- "x" => "\`",
- "r" => "/",
- "w" => "]",
- "W" => "]",
- "s" => ":",
- "i" => "]",
- "I" => "]"
- }
-
- PERCENT_PAREN = {
- "{" => "}",
- "[" => "]",
- "<" => ">",
- "(" => ")"
- }
-
- PERCENT_PAREN_REV = PERCENT_PAREN.invert
-
- Ltype2Token = {
- "\'" => TkSTRING,
- "\"" => TkSTRING,
- "\`" => TkXSTRING,
- "/" => TkREGEXP,
- "]" => TkDSTRING,
- ":" => TkSYMBOL
- }
- DLtype2Token = {
- "\"" => TkDSTRING,
- "\`" => TkDXSTRING,
- "/" => TkDREGEXP,
- }
-
- def lex_init()
- @OP = IRB::SLex.new
- @OP.def_rules("\0", "\004", "\032") do |op, io|
- Token(TkEND_OF_SCRIPT, '')
- end
-
- @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |op, io|
- @space_seen = true
- str = op
- while (ch = getc) =~ /[ \t\f\r\13]/ do
- str << ch
- end
- ungetc
- Token TkSPACE, str
- end
-
- @OP.def_rule("#") do |op, io|
- identify_comment
- end
-
- @OP.def_rule("=begin",
- proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do
- |op, io|
- @ltype = "="
- res = op
- until (ch = getc) == "\n" do
- res << ch
- end
- res << ch
-
- until ( peek_equal?("=end") && peek(4) =~ /\s/ ) do
- (ch = getc)
- res << ch
- end
-
- res << gets # consume =end
-
- @ltype = nil
- Token(TkRD_COMMENT, res)
- end
-
- @OP.def_rule("\n") do |op, io|
- print "\\n\n" if RDoc::RubyLex.debug?
- unless @heredoc_queue.empty?
- info = @heredoc_queue[0]
- if !info[:started] # "\n"
- info[:started] = true
- ungetc "\n"
- elsif info[:heredoc_end].nil? # heredoc body
- tk, heredoc_end = identify_here_document_body(info[:quoted], info[:lt], info[:indent])
- info[:heredoc_end] = heredoc_end
- ungetc "\n"
- else # heredoc end
- @heredoc_queue.shift
- @lex_state = :EXPR_BEG
- tk = Token(TkHEREDOCEND, info[:heredoc_end])
- if !@heredoc_queue.empty?
- @heredoc_queue[0][:started] = true
- ungetc "\n"
- end
- end
- end
- unless tk
- case @lex_state
- when :EXPR_BEG, :EXPR_FNAME, :EXPR_DOT
- @continue = true
- else
- @continue = false
- @lex_state = :EXPR_BEG unless @escaped_nl
- until (@indent_stack.empty? ||
- [TkLPAREN, TkLBRACK, TkLBRACE,
- TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
- @indent_stack.pop
- end
- end
- @current_readed = @readed
- @here_readed.clear
- tk = Token(TkNL)
- end
- @escaped_nl = false
- tk
- end
-
- @OP.def_rules("=") do
- |op, io|
- case @lex_state
- when :EXPR_FNAME, :EXPR_DOT
- @lex_state = :EXPR_ARG
- else
- @lex_state = :EXPR_BEG
- end
- Token(op)
- end
-
- @OP.def_rules("*", "**",
- "==", "===",
- "=~", "<=>",
- "<", "<=",
- ">", ">=", ">>", "=>") do
- |op, io|
- case @lex_state
- when :EXPR_FNAME, :EXPR_DOT
- tk = Token(TkId, op)
- @lex_state = :EXPR_ARG
- else
- tk = Token(op)
- @lex_state = :EXPR_BEG
- end
- tk
- end
-
- @OP.def_rules("->") do
- |op, io|
- @lex_state = :EXPR_ENDFN
- Token(op)
- end
-
- @OP.def_rules("!", "!=", "!~") do
- |op, io|
- case @lex_state
- when :EXPR_FNAME, :EXPR_DOT
- @lex_state = :EXPR_ARG
- Token(TkId, op)
- else
- @lex_state = :EXPR_BEG
- Token(op)
- end
- end
-
- @OP.def_rules("<<") do
- |op, io|
- tk = nil
- if @lex_state != :EXPR_END && @lex_state != :EXPR_CLASS &&
- (@lex_state != :EXPR_ARG || @space_seen)
- c = peek(0)
- if /\S/ =~ c && (/["'`]/ =~ c || /\w/ =~ c || c == "-" || c == "~")
- tk = identify_here_document(op)
- end
- end
- unless tk
- case @lex_state
- when :EXPR_FNAME, :EXPR_DOT
- tk = Token(TkId, op)
- @lex_state = :EXPR_ARG
- else
- tk = Token(op)
- @lex_state = :EXPR_BEG
- end
- end
- tk
- end
-
- @OP.def_rules("'", '"') do
- |op, io|
- identify_string(op)
- end
-
- @OP.def_rules("`") do
- |op, io|
- if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
- @lex_state = :EXPR_ARG
- Token(TkId, op)
- else
- identify_string(op)
- end
- end
-
- @OP.def_rules('?') do
- |op, io|
- if @lex_state == :EXPR_END
- @lex_state = :EXPR_BEG
- @after_question = true
- Token(TkQUESTION)
- else
- ch = getc
- if @lex_state == :EXPR_ARG && ch =~ /\s/
- ungetc
- @lex_state = :EXPR_BEG;
- Token(TkQUESTION)
- else
- @lex_state = :EXPR_END
- ch << getc if "\\" == ch
- Token(TkCHAR, "?#{ch}")
- end
- end
- end
-
- @OP.def_rules("&&", "||") do
- |op, io|
- @lex_state = :EXPR_BEG
- Token(op)
- end
-
- @OP.def_rules("&", "|") do
- |op, io|
- case @lex_state
- when :EXPR_FNAME, :EXPR_DOT
- tk = Token(TkId, op)
- @lex_state = :EXPR_ARG
- else
- tk = Token(op)
- @lex_state = :EXPR_BEG
- end
- tk
- end
-
- @OP.def_rules("+=", "-=", "*=", "**=",
- "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
- |op, io|
- @lex_state = :EXPR_BEG
- op =~ /^(.*)=$/
- Token(TkOPASGN, $1)
- end
-
- @OP.def_rule("+@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
- |op, io|
- @lex_state = :EXPR_ARG
- Token(TkId, op)
- end
-
- @OP.def_rule("-@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
- |op, io|
- @lex_state = :EXPR_ARG
- Token(TkId, op)
- end
-
- @OP.def_rules("+", "-") do
- |op, io|
- catch(:RET) do
- if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
- tk = Token(TkId, op)
- @lex_state = :EXPR_ARG
- elsif @lex_state == :EXPR_ARG
- if @space_seen and peek(0) =~ /[0-9]/
- throw :RET, identify_number(op)
- else
- @lex_state = :EXPR_BEG
- end
- elsif @lex_state != :EXPR_END and peek(0) =~ /[0-9]/
- throw :RET, identify_number(op)
- else
- @lex_state = :EXPR_BEG
- end
- tk = Token(op) unless tk
- tk
- end
- end
-
- @OP.def_rules(".", "&.") do
- |op, io|
- @lex_state = :EXPR_BEG
- if peek(0) =~ /[0-9]/
- ungetc
- identify_number
- else
- # for "obj.if" or "obj&.if" etc.
- @lex_state = :EXPR_DOT
- Token(op)
- end
- end
-
- @OP.def_rules("..", "...") do
- |op, io|
- @lex_state = :EXPR_BEG
- Token(op)
- end
-
- lex_int2
- end
-
- def lex_int2
- @OP.def_rules("]", "}", ")") do
- |op, io|
- @lex_state = :EXPR_END
- @indent -= 1
- @indent_stack.pop
- Token(op)
- end
-
- @OP.def_rule(":") do
- |op, io|
- if @lex_state == :EXPR_END || peek(0) =~ /\s/
- @lex_state = :EXPR_BEG
- Token(TkCOLON)
- else
- @lex_state = :EXPR_FNAME;
- Token(TkSYMBEG)
- end
- end
-
- @OP.def_rule("::") do
- |op, io|
- # p @lex_state.id2name, @space_seen
- if @lex_state == :EXPR_BEG or @lex_state == :EXPR_ARG && @space_seen
- @lex_state = :EXPR_BEG
- Token(TkCOLON3)
- else
- @lex_state = :EXPR_DOT
- Token(TkCOLON2)
- end
- end
-
- @OP.def_rule("/") do
- |op, io|
- if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
- @lex_state = :EXPR_ARG
- Token(TkId, op)
- elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID || @first_in_method_statement
- identify_string(op)
- elsif peek(0) == '='
- getc
- @lex_state = :EXPR_BEG
- Token(TkOPASGN, "/") #/)
- elsif @lex_state == :EXPR_ARG and @space_seen and peek(0) !~ /\s/
- identify_string(op)
- else
- @lex_state = :EXPR_BEG
- Token("/") #/)
- end
- end
-
- @OP.def_rules("^") do
- |op, io|
- case @lex_state
- when :EXPR_FNAME, :EXPR_DOT
- tk = Token(TkId, op)
- @lex_state = :EXPR_ARG
- else
- tk = Token(op)
- @lex_state = :EXPR_BEG
- end
- tk
- end
-
- # @OP.def_rules("^=") do
- # @lex_state = :EXPR_BEG
- # Token(OP_ASGN, :^)
- # end
-
- @OP.def_rules(",") do
- |op, io|
- @lex_state = :EXPR_BEG
- Token(op)
- end
-
- @OP.def_rules(";") do
- |op, io|
- @lex_state = :EXPR_BEG
- until (@indent_stack.empty? ||
- [TkLPAREN, TkLBRACK, TkLBRACE,
- TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
- @indent_stack.pop
- end
- Token(op)
- end
-
- @OP.def_rule("~") do
- |op, io|
- case @lex_state
- when :EXPR_FNAME, :EXPR_DOT
- @lex_state = :EXPR_ARG
- Token(TkId, op)
- else
- @lex_state = :EXPR_BEG
- Token(op)
- end
- end
-
- @OP.def_rule("~@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
- |op, io|
- @lex_state = :EXPR_BEG
- Token("~")
- end
-
- @OP.def_rule("(") do
- |op, io|
- @indent += 1
- if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
- @lex_state = :EXPR_BEG
- tk_c = TkfLPAREN
- else
- @lex_state = :EXPR_BEG
- tk_c = TkLPAREN
- end
- @indent_stack.push tk_c
- Token tk_c
- end
-
- @OP.def_rule("[]", proc{|op, io| @lex_state == :EXPR_FNAME}) do
- |op, io|
- @lex_state = :EXPR_ARG
- Token(TkId, op)
- end
-
- @OP.def_rule("[]=", proc{|op, io| @lex_state == :EXPR_FNAME}) do
- |op, io|
- @lex_state = :EXPR_ARG
- Token(TkId, op)
- end
-
- @OP.def_rule("[") do
- |op, io|
- text = nil
- @indent += 1
- if @lex_state == :EXPR_FNAME
- tk_c = TkfLBRACK
- else
- if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
- tk_c = TkLBRACK
- elsif @lex_state == :EXPR_ARG && @space_seen
- tk_c = TkLBRACK
- elsif @lex_state == :EXPR_DOT
- if peek(0) == "]"
- tk_c = TkIDENTIFIER
- getc
- if peek(0) == "="
- text = "[]="
- else
- text = "[]"
- end
- else
- tk_c = TkOp
- end
- else
- tk_c = TkfLBRACK
- end
- @lex_state = :EXPR_BEG
- end
- @indent_stack.push tk_c
- Token(tk_c, text)
- end
-
- @OP.def_rule("{") do
- |op, io|
- @indent += 1
- if @lex_state != :EXPR_END && @lex_state != :EXPR_ARG
- tk_c = TkLBRACE
- else
- tk_c = TkfLBRACE
- end
- @lex_state = :EXPR_BEG
- @indent_stack.push tk_c
- Token(tk_c)
- end
-
- @OP.def_rule('\\') do
- |op, io|
- if peek(0) == "\n"
- @space_seen = true
- @continue = true
- @escaped_nl = true
- end
- Token("\\")
- end
-
- @OP.def_rule('%') do
- |op, io|
- if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
- @lex_state = :EXPR_ARG
- Token(TkId, op)
- elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
- identify_quotation
- elsif peek(0) == '='
- getc
- @lex_state = :EXPR_BEG
- Token(TkOPASGN, '%')
- elsif @lex_state == :EXPR_ARG and @space_seen and peek(0) !~ /\s/
- identify_quotation
- else
- @lex_state = :EXPR_BEG
- Token("%") #))
- end
- end
-
- @OP.def_rule('$') do
- |op, io|
- identify_gvar
- end
-
- @OP.def_rule('@') do
- |op, io|
- if peek(0) =~ /[\w@]/
- ungetc
- identify_identifier
- else
- Token("@")
- end
- end
-
- # @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
- # |op, io|
- # @indent += 1
- # @lex_state = :EXPR_FNAME
- # # @lex_state = :EXPR_END
- # # until @rests[0] == "\n" or @rests[0] == ";"
- # # rests.shift
- # # end
- # end
-
- @OP.def_rule("_") do
- if peek_match?(/_END__/) and @lex_state == :EXPR_BEG then
- 6.times { getc }
- Token(TkEND_OF_SCRIPT, '__END__')
- else
- ungetc
- identify_identifier
- end
- end
-
- @OP.def_rule("") do
- |op, io|
- printf "MATCH: start %s: %s\n", op, io.inspect if RDoc::RubyLex.debug?
- if peek(0) =~ /[0-9]/
- t = identify_number
- else
- t = identify_identifier
- end
- printf "MATCH: end %s: %s\n", op, io.inspect if RDoc::RubyLex.debug?
- t
- end
-
- p @OP if RDoc::RubyLex.debug?
- end
-
- def identify_gvar
- @lex_state = :EXPR_END
-
- case ch = getc
- when /[~_*$?!@\/\\;,=:<>".]/ #"
- Token(TkGVAR, "$" + ch)
- when "-"
- Token(TkGVAR, "$-" + getc)
- when "&", "`", "'", "+"
- Token(TkBACK_REF, "$"+ch)
- when /[1-9]/
- ref = ch
- while (ch = getc) =~ /[0-9]/ do ref << ch end
- ungetc
- Token(TkNTH_REF, "$#{ref}")
- when /\w/
- ungetc
- ungetc
- identify_identifier
- else
- ungetc
- Token("$")
- end
- end
-
- IDENT_RE = eval '/[\w\u{0080}-\u{FFFFF}]/u'
-
- def identify_identifier
- token = ""
- if peek(0) =~ /[$@]/
- token.concat(c = getc)
- if c == "@" and peek(0) == "@"
- token.concat getc
- end
- end
-
- while (ch = getc) =~ IDENT_RE do
- print " :#{ch}: " if RDoc::RubyLex.debug?
- token.concat ch
- end
-
- ungetc
-
- if ((ch == "!" && peek(1) != "=") || ch == "?") && token[0,1] =~ /\w/
- token.concat getc
- end
-
- # almost fix token
-
- case token
- when /^\$/
- return Token(TkGVAR, token)
- when /^\@\@/
- @lex_state = :EXPR_END
- # p Token(TkCVAR, token)
- return Token(TkCVAR, token)
- when /^\@/
- @lex_state = :EXPR_END
- return Token(TkIVAR, token)
- end
-
- if @lex_state != :EXPR_DOT
- print token, "\n" if RDoc::RubyLex.debug?
-
- token_c, *trans = TkReading2Token[token]
- if token_c
- # reserved word?
-
- if (@lex_state != :EXPR_BEG &&
- @lex_state != :EXPR_FNAME &&
- trans[1])
- # modifiers
- token_c = TkSymbol2Token[trans[1]]
- @lex_state = trans[0]
- else
- if @lex_state != :EXPR_FNAME
- if ENINDENT_CLAUSE.include?(token)
- valid = peek(0) != ':'
-
- # check for ``class = val'' etc.
- case token
- when "class"
- valid = false unless peek_match?(/^\s*(<<|\w|::)/)
- when "def"
- valid = false if peek_match?(/^\s*(([+-\/*&\|^]|<<|>>|\|\||\&\&)=|\&\&|\|\|)/)
- when "do"
- valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&)/)
- when *ENINDENT_CLAUSE
- valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&|\|)/)
- else
- # no nothing
- end if valid
-
- if valid
- if token == "do"
- if ![TkFOR, TkWHILE, TkUNTIL].include?(@indent_stack.last)
- @indent += 1
- @indent_stack.push token_c
- end
- else
- @indent += 1
- @indent_stack.push token_c
- end
- else
- token_c = TkIDENTIFIER
- end
-
- elsif DEINDENT_CLAUSE.include?(token)
- @indent -= 1
- @indent_stack.pop
- end
- @lex_state = trans[0]
- else
- @lex_state = :EXPR_END
- end
- end
- if token_c.ancestors.include?(TkId) and peek(0) == ':' and !peek_match?(/^::/)
- token.concat getc
- token_c = TkSYMBOL
- end
- return Token(token_c, token)
- end
- end
-
- if @lex_state == :EXPR_FNAME
- @lex_state = :EXPR_END
- if peek(0) == '=' and peek(1) != '>'
- token.concat getc
- end
- elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_DOT ||
- @lex_state == :EXPR_ARG || @lex_state == :EXPR_MID
- @lex_state = :EXPR_ARG
- else
- @lex_state = :EXPR_END
- end
-
- if token[0, 1] =~ /[A-Z]/
- if token[-1] =~ /[!?]/
- token_c = TkIDENTIFIER
- else
- token_c = TkCONSTANT
- end
- elsif token[token.size - 1, 1] =~ /[!?]/
- token_c = TkFID
- else
- token_c = TkIDENTIFIER
- end
- if peek(0) == ':' and !peek_match?(/^::/)
- token.concat getc
- return Token(TkSYMBOL, token)
- else
- return Token(token_c, token)
- end
- end
-
- def identify_here_document(op)
- ch = getc
- start_token = op
- # if lt = PERCENT_LTYPE[ch]
- if ch == "-" or ch == "~"
- start_token.concat ch
- ch = getc
- indent = true
- end
- if /['"`]/ =~ ch
- start_token.concat ch
- user_quote = lt = ch
- quoted = ""
- while (c = getc) && c != lt
- quoted.concat c
- end
- start_token.concat quoted
- start_token.concat lt
- else
- user_quote = nil
- lt = '"'
- quoted = ch.dup
- while (c = getc) && c =~ /\w/
- quoted.concat c
- end
- start_token.concat quoted
- ungetc
- end
-
- @heredoc_queue << {
- quoted: quoted,
- lt: lt,
- indent: indent,
- started: false
- }
- @lex_state = :EXPR_END
- Token(RDoc::RubyLex::TkHEREDOCBEG, start_token)
- end
-
- def identify_here_document_body(quoted, lt, indent)
- ltback, @ltype = @ltype, lt
-
- doc = ""
- heredoc_end = nil
- while l = gets
- l = l.sub(/(:?\r)?\n\z/, "\n")
- if (indent ? l.strip : l.chomp) == quoted
- heredoc_end = l
- break
- end
- doc << l
- end
- raise Error, "Missing terminating #{quoted} for string" unless heredoc_end
-
- @ltype = ltback
- @lex_state = :EXPR_BEG
- [Token(RDoc::RubyLex::TkHEREDOC, doc), heredoc_end]
- end
-
- def identify_quotation
- type = ch = getc
- if lt = PERCENT_LTYPE[type]
- ch = getc
- elsif type =~ /\W/
- type = nil
- lt = "\""
- else
- return Token(TkMOD, '%')
- end
- # if ch !~ /\W/
- # ungetc
- # next
- # end
- #@ltype = lt
- @quoted = ch unless @quoted = PERCENT_PAREN[ch]
- identify_string(lt, @quoted, type)
- end
-
- def identify_number(op = "")
- @lex_state = :EXPR_END
-
- num = op
-
- if peek(0) == "0" && peek(1) !~ /[.eEri]/
- num << getc
-
- case peek(0)
- when /[xX]/
- ch = getc
- match = /[0-9a-fA-F_]/
- when /[bB]/
- ch = getc
- match = /[01_]/
- when /[oO]/
- ch = getc
- match = /[0-7_]/
- when /[dD]/
- ch = getc
- match = /[0-9_]/
- when /[0-7]/
- match = /[0-7_]/
- when /[89]/
- raise Error, "Illegal octal digit"
- else
- return Token(TkINTEGER, num)
- end
-
- num << ch if ch
-
- len0 = true
- non_digit = false
- while ch = getc
- num << ch
- if match =~ ch
- if ch == "_"
- if non_digit
- raise Error, "trailing `#{ch}' in number"
- else
- non_digit = ch
- end
- else
- non_digit = false
- len0 = false
- end
- else
- ungetc
- num[-1, 1] = ''
- if len0
- raise Error, "numeric literal without digits"
- end
- if non_digit
- raise Error, "trailing `#{non_digit}' in number"
- end
- break
- end
- end
- return Token(TkINTEGER, num)
- end
-
- type = TkINTEGER
- allow_point = true
- allow_e = true
- allow_ri = true
- non_digit = false
- while ch = getc
- num << ch
- case ch
- when /[0-9]/
- non_digit = false
- when "_"
- non_digit = ch
- when allow_point && "."
- if non_digit
- raise Error, "trailing `#{non_digit}' in number"
- end
- type = TkFLOAT
- if peek(0) !~ /[0-9]/
- type = TkINTEGER
- ungetc
- num[-1, 1] = ''
- break
- end
- allow_point = false
- when allow_e && "e", allow_e && "E"
- if non_digit
- raise Error, "trailing `#{non_digit}' in number"
- end
- type = TkFLOAT
- if peek(0) =~ /[+-]/
- num << getc
- end
- allow_e = false
- allow_ri = false
- allow_point = false
- non_digit = ch
- when allow_ri && "r"
- if non_digit
- raise Error, "trailing `#{non_digit}' in number"
- end
- type = TkRATIONAL
- if peek(0) == 'i'
- type = TkIMAGINARY
- num << getc
- end
- break
- when allow_ri && "i"
- if non_digit && non_digit != "r"
- raise Error, "trailing `#{non_digit}' in number"
- end
- type = TkIMAGINARY
- break
- else
- if non_digit
- raise Error, "trailing `#{non_digit}' in number"
- end
- ungetc
- num[-1, 1] = ''
- break
- end
- end
-
- Token(type, num)
- end
-
- def identify_string(ltype, quoted = ltype, type = nil)
- close = PERCENT_PAREN.values.include?(quoted)
- @ltype = ltype
- @quoted = quoted
-
- str = if ltype == quoted and %w[" ' / `].include? ltype and type.nil? then
- ltype.dup
- else
- "%#{type}#{PERCENT_PAREN_REV[quoted]||quoted}"
- end
-
- subtype = nil
- begin
- nest = 0
-
- while ch = getc
- str << ch
-
- if @quoted == ch and nest <= 0
- break
- elsif @ltype != "'" && @ltype != "]" && @ltype != ":" and ch == "#"
- ch = getc
- if ch == "{" then
- subtype = true
- str << ch << skip_inner_expression
- next
- else
- ungetc
- end
- elsif ch == '\\'
- case @ltype
- when "'" then
- case ch = getc
- when "'", '\\' then
- str << ch
- else
- str << ch
- end
- else
- str << read_escape
- end
- end
-
- if close then
- if PERCENT_PAREN[ch] == @quoted
- nest += 1
- elsif ch == @quoted
- nest -= 1
- end
- end
- end
-
- if @ltype == "/"
- while peek(0) =~ /i|m|x|o|e|s|u|n/
- str << getc
- end
- end
-
- if peek(0) == ':' and !peek_match?(/^::/) and :EXPR_BEG == @lex_state and !@after_question
- str.concat getc
- return Token(TkSYMBOL, str)
- elsif subtype
- Token(DLtype2Token[ltype], str)
- else
- Token(Ltype2Token[ltype], str)
- end
- ensure
- @ltype = nil
- @quoted = nil
- @lex_state = :EXPR_END
- end
- end
-
- def skip_inner_expression
- res = ""
- nest = 0
- while ch = getc
- res << ch
- if ch == '}'
- break if nest.zero?
- nest -= 1
- elsif ch == '{'
- nest += 1
- end
- end
- res
- end
-
- def identify_comment
- @ltype = "#"
-
- comment = '#'
-
- while ch = getc
- # if ch == "\\" #"
- # read_escape
- # end
- if ch == "\n"
- @ltype = nil
- ungetc
- break
- end
-
- comment << ch
- end
-
- return Token(TkCOMMENT, comment)
- end
-
- def read_escape
- escape = ''
- ch = getc
-
- case ch
- when "\n", "\r", "\f"
- escape << ch
- when "\\", "n", "t", "r", "f", "v", "a", "e", "b", "s" #"
- escape << ch
- when /[0-7]/
- ungetc ch
- 3.times do
- ch = getc
- case ch
- when /[0-7]/
- escape << ch
- when nil
- break
- else
- ungetc
- break
- end
- end
-
- when "x"
- escape << ch
-
- 2.times do
- ch = getc
- case ch
- when /[0-9a-fA-F]/
- escape << ch
- when nil
- break
- else
- ungetc
- break
- end
- end
-
- when "M"
- escape << ch
-
- ch = getc
- if ch != '-'
- ungetc
- else
- escape << ch
-
- ch = getc
- if ch == "\\" #"
- ungetc
- escape << read_escape
- else
- escape << ch
- end
- end
-
- when "C", "c" #, "^"
- escape << ch
-
- if ch == "C"
- ch = getc
-
- if ch == "-"
- escape << ch
- ch = getc
- escape << ch
-
- escape << read_escape if ch == "\\"
- else
- ungetc
- end
- elsif (ch = getc) == "\\" #"
- escape << ch << read_escape
- end
- else
- escape << ch
-
- # other characters
- end
-
- escape
- end
-
- # :startdoc:
-
-end
-
-#RDoc::RubyLex.debug_level = 1
diff --git a/lib/rdoc/stats/normal.rb b/lib/rdoc/stats/normal.rb
index ba00b6cbdf..8ad0e453c6 100644
--- a/lib/rdoc/stats/normal.rb
+++ b/lib/rdoc/stats/normal.rb
@@ -1,5 +1,5 @@
# frozen_string_literal: false
-require 'io/console/size'
+require 'io/console'
##
# Stats printer that prints just the files being documented with a progress
@@ -23,7 +23,7 @@ class RDoc::Stats::Normal < RDoc::Stats::Quiet
# Print a progress bar, but make sure it fits on a single line. Filename
# will be truncated if necessary.
- terminal_width = IO.console_size[1].to_i.nonzero? || 80
+ terminal_width = IO.console.winsize[1].to_i.nonzero? || 80
max_filename_size = terminal_width - progress_bar.size
if filename.size > max_filename_size then
diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb
index 0cfa2f1384..df5232f7ea 100644
--- a/lib/rdoc/token_stream.rb
+++ b/lib/rdoc/token_stream.rb
@@ -30,28 +30,41 @@ module RDoc::TokenStream
token_stream.map do |t|
next unless t
- style = case t
- when RDoc::RubyToken::TkCONSTANT then 'ruby-constant'
- when RDoc::RubyToken::TkKW then 'ruby-keyword'
- when RDoc::RubyToken::TkIVAR then 'ruby-ivar'
- when RDoc::RubyToken::TkOp then 'ruby-operator'
- when RDoc::RubyToken::TkId then 'ruby-identifier'
- when RDoc::RubyToken::TkREGEXP then 'ruby-regexp'
- when RDoc::RubyToken::TkDREGEXP then 'ruby-regexp'
- when RDoc::RubyToken::TkNode then 'ruby-node'
- when RDoc::RubyToken::TkCOMMENT then 'ruby-comment'
- when RDoc::RubyToken::TkXSTRING then 'ruby-string'
- when RDoc::RubyToken::TkSTRING then 'ruby-string'
- when RDoc::RubyToken::TkVal then 'ruby-value'
+ style = case t[:kind]
+ when :on_const then 'ruby-constant'
+ when :on_kw then 'ruby-keyword'
+ when :on_ivar then 'ruby-ivar'
+ when :on_cvar then 'ruby-identifier'
+ when :on_gvar then 'ruby-identifier'
+ when '=' != t[:text] && :on_op then
+ if RDoc::RipperStateLex::EXPR_ARG == t[:state] then
+ 'ruby-identifier'
+ else
+ 'ruby-operator'
+ end
+ when :on_tlambda then 'ruby-operator'
+ when :on_ident then 'ruby-identifier'
+ when :on_label then 'ruby-value'
+ when :on_backref, :on_dstring
+ then 'ruby-node'
+ when :on_comment then 'ruby-comment'
+ when :on_embdoc then 'ruby-comment'
+ when :on_regexp then 'ruby-regexp'
+ when :on_tstring then 'ruby-string'
+ when :on_int, :on_float,
+ :on_rational, :on_imaginary,
+ :on_heredoc,
+ :on_symbol, :on_CHAR then 'ruby-value'
+ when :on_heredoc_beg, :on_heredoc_end
+ then 'ruby-identifier'
end
comment_with_nl = false
- case t
- when RDoc::RubyToken::TkRD_COMMENT, RDoc::RubyToken::TkHEREDOCEND
- comment_with_nl = true if t.text =~ /\n$/
- text = t.text.rstrip
+ if :on_comment == t[:kind] or :on_embdoc == t[:kind] or :on_heredoc_end == t[:kind]
+ comment_with_nl = true if "\n" == t[:text][-1]
+ text = t[:text].rstrip
else
- text = t.text
+ text = t[:text]
end
text = CGI.escapeHTML text
diff --git a/test/rdoc/test_rdoc_any_method.rb b/test/rdoc/test_rdoc_any_method.rb
index fdbb62efa6..fdfb0f3995 100644
--- a/test/rdoc/test_rdoc_any_method.rb
+++ b/test/rdoc/test_rdoc_any_method.rb
@@ -74,7 +74,7 @@ method(a, b) { |c, d| ... }
def test_markup_code
tokens = [
- RDoc::RubyToken::TkCONSTANT. new(0, 0, 0, 'CONSTANT'),
+ { :line_no => 0, :char_no => 0, :kind => :on_const, :text => 'CONSTANT' },
]
@c2_a.collect_tokens
diff --git a/test/rdoc/test_rdoc_constant.rb b/test/rdoc/test_rdoc_constant.rb
index c43aa7dc2b..a03ab97a7d 100644
--- a/test/rdoc/test_rdoc_constant.rb
+++ b/test/rdoc/test_rdoc_constant.rb
@@ -86,7 +86,7 @@ class TestRDocConstant < XrefTestCase
assert_equal top_level, loaded.file
assert_equal 'Klass::CONST', loaded.full_name
assert_equal 'CONST', loaded.name
- assert_nil loaded.visibility
+ assert_equal :public, loaded.visibility
assert_equal cm, loaded.parent
assert_equal section, loaded.section
end
@@ -114,7 +114,7 @@ class TestRDocConstant < XrefTestCase
assert_equal top_level, loaded.file
assert_equal 'Klass::CONST', loaded.full_name
assert_equal 'CONST', loaded.name
- assert_nil loaded.visibility
+ assert_equal :public, loaded.visibility
assert_equal cm, loaded.parent
assert_equal section, loaded.section
@@ -146,7 +146,7 @@ class TestRDocConstant < XrefTestCase
assert_equal top_level, loaded.file
assert_equal 'Klass::CONST', loaded.full_name
assert_equal 'CONST', loaded.name
- assert_nil loaded.visibility
+ assert_equal :public, loaded.visibility
assert_equal cm, loaded.parent
assert_equal section, loaded.section
diff --git a/test/rdoc/test_rdoc_context.rb b/test/rdoc/test_rdoc_context.rb
index 54e365f7f4..ceac14ac90 100644
--- a/test/rdoc/test_rdoc_context.rb
+++ b/test/rdoc/test_rdoc_context.rb
@@ -719,6 +719,7 @@ class TestRDocContext < XrefTestCase
assert_equal [@pub, @prot, @priv], @vis.method_list
assert_equal [@apub, @aprot, @apriv], @vis.attributes
+ assert_equal [@cpub, @cpriv], @vis.constants
end
def test_remove_invisible_nodoc
@@ -728,6 +729,7 @@ class TestRDocContext < XrefTestCase
assert_equal [@pub, @prot, @priv], @vis.method_list
assert_equal [@apub, @aprot, @apriv], @vis.attributes
+ assert_equal [@cpub, @cpriv], @vis.constants
end
def test_remove_invisible_protected
@@ -737,6 +739,7 @@ class TestRDocContext < XrefTestCase
assert_equal [@pub, @prot], @vis.method_list
assert_equal [@apub, @aprot], @vis.attributes
+ assert_equal [@cpub], @vis.constants
end
def test_remove_invisible_public
@@ -746,6 +749,7 @@ class TestRDocContext < XrefTestCase
assert_equal [@pub], @vis.method_list
assert_equal [@apub], @vis.attributes
+ assert_equal [@cpub], @vis.constants
end
def test_remove_invisible_public_force
@@ -755,11 +759,13 @@ class TestRDocContext < XrefTestCase
@prot.force_documentation = true
@apriv.force_documentation = true
@aprot.force_documentation = true
+ @cpriv.force_documentation = true
@vis.remove_invisible :public
assert_equal [@pub, @prot, @priv], @vis.method_list
assert_equal [@apub, @aprot, @apriv], @vis.attributes
+ assert_equal [@cpub, @cpriv], @vis.constants
end
def test_remove_invisible_in_protected
@@ -922,6 +928,9 @@ class TestRDocContext < XrefTestCase
@aprot = RDoc::Attr.new nil, 'prot', 'RW', nil
@apriv = RDoc::Attr.new nil, 'priv', 'RW', nil
+ @cpub = RDoc::Constant.new 'CONST_PUBLIC', nil, nil
+ @cpriv = RDoc::Constant.new 'CONST_PRIVATE', nil, nil
+
@vis = RDoc::NormalClass.new 'Vis'
@vis.add_method @pub
@vis.add_method @prot
@@ -931,11 +940,16 @@ class TestRDocContext < XrefTestCase
@vis.add_attribute @aprot
@vis.add_attribute @apriv
+ @vis.add_constant @cpub
+ @vis.add_constant @cpriv
+
@prot.visibility = :protected
@priv.visibility = :private
@aprot.visibility = :protected
@apriv.visibility = :private
+
+ @cpriv.visibility = :private
end
end
diff --git a/test/rdoc/test_rdoc_markup_to_html.rb b/test/rdoc/test_rdoc_markup_to_html.rb
index fd8a7bb0ca..b5edeb04d2 100644
--- a/test/rdoc/test_rdoc_markup_to_html.rb
+++ b/test/rdoc/test_rdoc_markup_to_html.rb
@@ -452,6 +452,9 @@ class TestRDocMarkupToHtml < RDoc::Markup::FormatterTestCase
end
def test_accept_verbatim_nl_after_backslash
+ # TODO: Remove "skip" after the issue is resolved: https://github.com/jruby/jruby/issues/4787
+ # This "skip" is for strange behavior around escaped newline on JRuby
+ skip if defined? JRUBY_VERSION
verb = @RM::Verbatim.new("a = 1 if first_flag_var and \\\n", " this_is_flag_var\n")
@to.start_accepting
diff --git a/test/rdoc/test_rdoc_parser_c.rb b/test/rdoc/test_rdoc_parser_c.rb
index d0732597dc..5341da16e2 100644
--- a/test/rdoc/test_rdoc_parser_c.rb
+++ b/test/rdoc/test_rdoc_parser_c.rb
@@ -1037,7 +1037,7 @@ Init_Foo(void) {
other_function.comment.text
assert_equal '()', other_function.params
- code = other_function.token_stream.first.text
+ code = other_function.token_stream.first[:text]
assert_equal "VALUE\nother_function() {\n}", code
end
@@ -1107,7 +1107,7 @@ Init_Foo(void) {
other_function.comment.text
assert_equal '()', other_function.params
- code = other_function.token_stream.first.text
+ code = other_function.token_stream.first[:text]
assert_equal "VALUE\nother_function() {\n}", code
end
@@ -1141,7 +1141,7 @@ Init_Foo(void) {
assert_equal '()', other_function.params
assert_equal 8, other_function.line
- code = other_function.token_stream.first.text
+ code = other_function.token_stream.first[:text]
assert_equal "VALUE\nrb_other_function() {\n}", code
end
@@ -1174,7 +1174,7 @@ Init_Foo(void) {
assert_equal '()', other_function.params
assert_equal 4, other_function.line
- code = other_function.token_stream.first.text
+ code = other_function.token_stream.first[:text]
assert_equal "#define other_function rb_other_function", code
end
@@ -1314,7 +1314,7 @@ Init_Foo(void) {
other_function.comment.text
assert_equal '()', other_function.params
- code = other_function.token_stream.first.text
+ code = other_function.token_stream.first[:text]
assert_equal "DLL_LOCAL VALUE\nother_function() {\n}", code
end
diff --git a/test/rdoc/test_rdoc_parser_ruby.rb b/test/rdoc/test_rdoc_parser_ruby.rb
index 55c03d4628..cda407ed51 100644
--- a/test/rdoc/test_rdoc_parser_ruby.rb
+++ b/test/rdoc/test_rdoc_parser_ruby.rb
@@ -74,7 +74,7 @@ class C; end
comment = parser.collect_first_comment
- assert_equal RDoc::Comment.new("=begin\nfirst\n=end\n\n", @top_level), comment
+ assert_equal RDoc::Comment.new("=begin\nfirst\n=end\n", @top_level), comment
end
def test_get_class_or_module
@@ -84,7 +84,7 @@ class C; end
cont, name_t, given_name = util_parser('A') .get_class_or_module ctxt
assert_equal ctxt, cont
- assert_equal 'A', name_t.text
+ assert_equal 'A', name_t[:text]
assert_equal 'A', given_name
cont, name_t, given_name = util_parser('B::C') .get_class_or_module ctxt
@@ -92,16 +92,16 @@ class C; end
b = @store.find_module_named('B')
assert_equal b, cont
assert_equal [@top_level], b.in_files
- assert_equal 'C', name_t.text
+ assert_equal 'C', name_t[:text]
assert_equal 'B::C', given_name
cont, name_t, given_name = util_parser('D:: E').get_class_or_module ctxt
assert_equal @store.find_module_named('D'), cont
- assert_equal 'E', name_t.text
+ assert_equal 'E', name_t[:text]
assert_equal 'D::E', given_name
- assert_raises NoMethodError do
+ assert_raises RDoc::Error do
util_parser("A::\nB").get_class_or_module ctxt
end
end
@@ -1194,10 +1194,12 @@ EOF
assert_equal klass.current_section, foo.section
stream = [
- tk(:COMMENT, 0, 1, 1, nil,
- "# File #{@top_level.relative_name}, line 1"),
- RDoc::Parser::Ruby::NEWLINE_TOKEN,
- tk(:SPACE, 0, 1, 1, nil, ''),
+ {
+ :line_no => 1, :char_no => 1, :kind => :on_comment,
+ :text => "# File #{@top_level.relative_name}, line 1"
+ },
+ { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" },
+ { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' }
]
assert_equal stream, foo.token_stream
@@ -1358,6 +1360,33 @@ A::B::C = 1
assert_equal 'comment', c.comment
end
+ def test_parse_constant_with_bracket
+ util_parser <<-RUBY
+class Klass
+end
+
+class Klass2
+ CONSTANT = Klass
+end
+
+class Klass3
+ CONSTANT_2 = {}
+ CONSTANT_2[1] = Klass
+end
+ RUBY
+
+ @parser.scan
+
+ klass = @store.find_class_named 'Klass'
+ klass2 = @store.find_class_named 'Klass2'
+ klass3 = @store.find_class_named 'Klass3'
+ constant = klass2.find_module_named 'CONSTANT'
+ constant2 = klass3.find_module_named 'CONSTANT_2'
+ assert_equal klass, klass2.constants.first.is_alias_for
+ refute_equal klass, klass3.constants.first.is_alias_for
+ assert_nil klass3.find_module_named 'CONSTANT_2'
+ end
+
def test_parse_extend_or_include_extend
klass = RDoc::NormalClass.new 'C'
klass.parent = @top_level
@@ -1434,20 +1463,30 @@ A::B::C = 1
assert_equal klass.current_section, foo.section
stream = [
- tk(:COMMENT, 0, 1, 1, nil,
- "# File #{@top_level.relative_name}, line 1"),
- RDoc::Parser::Ruby::NEWLINE_TOKEN,
- tk(:SPACE, 0, 1, 1, nil, ''),
- tk(:IDENTIFIER, 0, 1, 0, 'add_my_method', 'add_my_method'),
- tk(:SPACE, 0, 1, 13, nil, ' '),
- tk(:SYMBOL, 0, 1, 14, nil, ':foo'),
- tk(:COMMA, 0, 1, 18, nil, ','),
- tk(:SPACE, 0, 1, 19, nil, ' '),
- tk(:SYMBOL, 0, 1, 20, nil, ':bar'),
- tk(:NL, 0, 1, 24, nil, "\n"),
+ {
+ :line_no => 1, :char_no => 1, :kind => :on_comment,
+ :text => "# File #{@top_level.relative_name}, line 1"
+ },
+ { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" },
+ { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' },
+ { :line_no => 1, :char_no => 0, :kind => :on_ident, :text => 'add_my_method' },
+ { :line_no => 1, :char_no => 13, :kind => :on_sp, :text => ' ' },
+ { :line_no => 1, :char_no => 14, :kind => :on_symbol, :text => ':foo' },
+ { :line_no => 1, :char_no => 18, :kind => :on_comma, :text => ',' },
+ { :line_no => 1, :char_no => 19, :kind => :on_sp, :text => ' ' },
+ { :line_no => 1, :char_no => 20, :kind => :on_symbol, :text => ':bar' },
+ { :line_no => 1, :char_no => 24, :kind => :on_nl, :text => "\n" }
]
+ parsed_stream = foo.token_stream.map { |t|
+ {
+ :line_no => t[:line_no],
+ :char_no => t[:char_no],
+ :kind => t[:kind],
+ :text => t[:text]
+ }
+ }
- assert_equal stream, foo.token_stream
+ assert_equal stream, parsed_stream
end
def test_parse_meta_method_block
@@ -1468,7 +1507,10 @@ end
@parser.parse_meta_method klass, RDoc::Parser::Ruby::NORMAL, tk, comment
- assert_equal tk(:NL, 0, 3, 3, 3, "\n"), @parser.get_tk
+ rest = { :line_no => 3, :char_no => 3, :kind => :on_nl, :text => "\n" }
+ tk = @parser.get_tk
+ tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => tk[:kind], :text => tk[:text] }
+ assert_equal rest, tk
end
def test_parse_meta_method_define_method
@@ -1631,23 +1673,30 @@ end
assert_equal klass.current_section, foo.section
stream = [
- tk(:COMMENT, 0, 1, 1, nil,
- "# File #{@top_level.relative_name}, line 1"),
- RDoc::Parser::Ruby::NEWLINE_TOKEN,
- tk(:SPACE, 0, 1, 1, nil, ''),
- tk(:DEF, 0, 1, 0, 'def', 'def'),
- tk(:SPACE, 3, 1, 3, nil, ' '),
- tk(:IDENTIFIER, 4, 1, 4, 'foo', 'foo'),
- tk(:LPAREN, 7, 1, 7, nil, '('),
- tk(:RPAREN, 8, 1, 8, nil, ')'),
- tk(:SPACE, 9, 1, 9, nil, ' '),
- tk(:COLON, 10, 1, 10, nil, ':'),
- tk(:IDENTIFIER, 11, 1, 11, 'bar', 'bar'),
- tk(:SPACE, 14, 1, 14, nil, ' '),
- tk(:END, 15, 1, 15, 'end', 'end'),
+ {
+ :line_no => 1, :char_no => 1, :kind => :on_comment,
+ :text => "# File #{@top_level.relative_name}, line 1" },
+ { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" },
+ { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' },
+ { :line_no => 1, :char_no => 0, :kind => :on_kw, :text => 'def' },
+ { :line_no => 1, :char_no => 3, :kind => :on_sp, :text => ' ' },
+ { :line_no => 1, :char_no => 4, :kind => :on_ident, :text => 'foo' },
+ { :line_no => 1, :char_no => 7, :kind => :on_lparen, :text => '(' },
+ { :line_no => 1, :char_no => 8, :kind => :on_rparen, :text => ')' },
+ { :line_no => 1, :char_no => 9, :kind => :on_sp, :text => ' ' },
+ { :line_no => 1, :char_no => 10, :kind => :on_symbol, :text => ':bar' },
+ { :line_no => 1, :char_no => 14, :kind => :on_sp, :text => ' ' },
+ { :line_no => 1, :char_no => 15, :kind => :on_kw, :text => 'end' }
]
-
- assert_equal stream, foo.token_stream
+ parsed_stream = foo.token_stream.map { |t|
+ {
+ :line_no => t[:line_no],
+ :char_no => t[:char_no],
+ :kind => t[:kind],
+ :text => t[:text]
+ }
+ }
+ assert_equal stream, parsed_stream
end
def test_parse_redefinable_methods
@@ -1664,8 +1713,8 @@ end
end
klass.method_list.each do |method|
- assert_kind_of RDoc::RubyToken::TkId, method.token_stream[5]
- assert_includes redefinable_ops, method.token_stream[5].text
+ assert_equal :on_ident, method.token_stream[5][:kind]
+ assert_includes redefinable_ops, method.token_stream[5][:text]
end
end
@@ -1909,6 +1958,20 @@ end
assert_equal '(arg1, arg2, arg3)', foo.params
end
+ def test_parse_method_parameters_with_paren_comment_continue
+ klass = RDoc::NormalClass.new 'Foo'
+ klass.parent = @top_level
+
+ util_parser "def foo(arg1, arg2, # some useful comment\narg3)\nend"
+
+ tk = @parser.get_tk
+
+ @parser.parse_method klass, RDoc::Parser::Ruby::NORMAL, tk, @comment
+
+ foo = klass.method_list.first
+ assert_equal '(arg1, arg2, arg3)', foo.params
+ end
+
def test_parse_method_star
klass = RDoc::NormalClass.new 'Foo'
klass.parent = @top_level
@@ -2091,22 +2154,34 @@ end
assert_equal 2, x.method_list.length
a = x.method_list.first
- expected = [
- tk(:COMMENT, 0, 2, 1, nil, "# File #{@filename}, line 2"),
- tk(:NL, 0, 0, 0, nil, "\n"),
- tk(:SPACE, 0, 1, 1, nil, ''),
- tk(:DEF, 8, 2, 0, 'def', 'def'),
- tk(:SPACE, 11, 2, 3, nil, ' '),
- tk(:IDENTIFIER, 12, 2, 4, 'a', 'a'),
- tk(:NL, 13, 2, 5, nil, "\n"),
- tk(:REGEXP, 14, 3, 0, nil, '%r{#}'),
- tk(:NL, 19, 3, 5, nil, "\n"),
- tk(:DREGEXP, 20, 4, 0, nil, '%r{#{}}'),
- tk(:NL, 27, 4, 7, nil, "\n"),
- tk(:END, 28, 5, 0, 'end', 'end'),
- ]
- assert_equal expected, a.token_stream
+ expected = [
+ {
+ :line_no => 2, :char_no => 1, :kind => :on_comment,
+ :text => "# File #{@filename}, line 2"
+ },
+ { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" },
+ { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' },
+ { :line_no => 2, :char_no => 0, :kind => :on_kw, :text => 'def' },
+ { :line_no => 2, :char_no => 3, :kind => :on_sp, :text => ' ' },
+ { :line_no => 2, :char_no => 4, :kind => :on_ident, :text => 'a' },
+ { :line_no => 2, :char_no => 5, :kind => :on_nl, :text => "\n" },
+ { :line_no => 3, :char_no => 0, :kind => :on_regexp, :text => '%r{#}' },
+ { :line_no => 3, :char_no => 5, :kind => :on_nl, :text => "\n" },
+ { :line_no => 4, :char_no => 0, :kind => :on_regexp, :text => '%r{#{}}' },
+ { :line_no => 4, :char_no => 7, :kind => :on_nl, :text => "\n" },
+ { :line_no => 5, :char_no => 0, :kind => :on_kw, :text => 'end' }
+ ]
+ parsed_stream = a.token_stream.map { |tk|
+ {
+ :line_no => tk[:line_no],
+ :char_no => tk[:char_no],
+ :kind => tk[:kind],
+ :text => tk[:text]
+ }
+ }
+
+ assert_equal expected, parsed_stream
end
def test_parse_statements_encoding
@@ -2286,6 +2361,9 @@ class Foo
SIXTH_CONSTANT = #{sixth_constant}
SEVENTH_CONSTANT = proc { |i| begin i end }
+
+ EIGHTH_CONSTANT = "a" \\
+ "b"
end
EOF
@@ -2331,6 +2409,11 @@ EOF
assert_equal 'SEVENTH_CONSTANT', constant.name
assert_equal "proc { |i| begin i end }", constant.value
assert_equal @top_level, constant.file
+
+ constant = constants[7]
+ assert_equal 'EIGHTH_CONSTANT', constant.name
+ assert_equal "\"a\" \\\n\"b\"", constant.value
+ assert_equal @top_level, constant.file
end
def test_parse_statements_identifier_attr
@@ -2504,7 +2587,7 @@ EXPTECTED
util_parser <def blah()
- <<~EOM if true
+ <<-EOM if true
EOM
end
EXPTECTED
@@ -2525,7 +2608,7 @@ EXPTECTED
blah = foo.method_list.first
markup_code = blah.markup_code.sub(/^.*\n/, '')
- assert_equal markup_code, expected
+ assert_equal expected, markup_code
end
def test_parse_statements_method_oneliner_with_regexp
@@ -2770,17 +2853,21 @@ RUBY
end
def test_parse_symbol_in_arg
- util_parser ':blah "blah" "#{blah}" blah'
+ util_parser '[:blah, "blah", "#{blah}", blah]'
+ @parser.get_tk # skip '['
assert_equal 'blah', @parser.parse_symbol_in_arg
+ @parser.get_tk # skip ','
@parser.skip_tkspace
assert_equal 'blah', @parser.parse_symbol_in_arg
+ @parser.get_tk # skip ','
@parser.skip_tkspace
assert_equal nil, @parser.parse_symbol_in_arg
+ @parser.get_tk # skip ','
@parser.skip_tkspace
@@ -2888,7 +2975,7 @@ end
assert_equal 'category', directive
assert_equal 'test', value
- assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk
+ assert_equal nil, parser.get_tk
end
def test_read_directive_allow
@@ -2898,7 +2985,7 @@ end
assert_nil directive
- assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk
+ assert_equal nil, parser.get_tk
end
def test_read_directive_empty
@@ -2908,7 +2995,7 @@ end
assert_nil directive
- assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk
+ assert_equal nil, parser.get_tk
end
def test_read_directive_no_comment
@@ -2918,18 +3005,18 @@ end
assert_nil directive
- assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk
+ assert_equal nil, parser.get_tk
end
def test_read_directive_one_liner
- parser = util_parser '; end # :category: test'
+ parser = util_parser 'AAA = 1 # :category: test'
directive, value = parser.read_directive %w[category]
assert_equal 'category', directive
assert_equal 'test', value
- assert_kind_of RDoc::RubyToken::TkSEMICOLON, parser.get_tk
+ assert_equal :on_const, parser.get_tk[:kind]
end
def test_read_documentation_modifiers
@@ -2974,10 +3061,10 @@ end
def test_sanity_integer
util_parser '1'
- assert_equal '1', @parser.get_tk.text
+ assert_equal '1', @parser.get_tk[:text]
util_parser '1.0'
- assert_equal '1.0', @parser.get_tk.text
+ assert_equal '1.0', @parser.get_tk[:text]
end
def test_sanity_interpolation
@@ -2986,7 +3073,7 @@ end
while tk = @parser.get_tk do last_tk = tk end
- assert_equal "\n", last_tk.text
+ assert_equal 'end', last_tk[:text]
end
# If you're writing code like this you're doing it wrong
@@ -2994,15 +3081,15 @@ end
def test_sanity_interpolation_crazy
util_parser '"#{"#{"a")}" if b}"'
- assert_equal '"#{"#{"a")}" if b}"', @parser.get_tk.text
- assert_equal RDoc::RubyToken::TkNL, @parser.get_tk.class
+ assert_equal '"#{"#{"a")}" if b}"', @parser.get_tk[:text]
+ assert_equal nil, @parser.get_tk
end
def test_sanity_interpolation_curly
util_parser '%{ #{} }'
- assert_equal '%{ #{} }', @parser.get_tk.text
- assert_equal RDoc::RubyToken::TkNL, @parser.get_tk.class
+ assert_equal '%{ #{} }', @parser.get_tk[:text]
+ assert_equal nil, @parser.get_tk
end
def test_sanity_interpolation_format
@@ -3537,6 +3624,53 @@ end
assert_equal 2, public_method_count
end
+ def test_scan_constant_visibility
+ util_parser <<-RUBY
+class C
+ CONST_A = 123
+
+ CONST_B = 234
+ private_constant :CONST_B
+
+ CONST_C = 345
+ public_constant :CONST_C
+end
+ RUBY
+
+ @parser.scan
+
+ c = @store.find_class_named 'C'
+ const_a, const_b, const_c, const_d = c.constants.sort_by(&:name)
+
+ assert_equal 'CONST_A', const_a.name
+ assert_equal :public, const_a.visibility
+
+ assert_equal 'CONST_B', const_b.name
+ assert_equal :private, const_b.visibility
+
+ assert_equal 'CONST_C', const_c.name
+ assert_equal :public, const_c.visibility
+ end
+
+ def test_document_after_rescue_inside_paren
+ util_parser <<-RUBY
+class C
+ attr_accessor :sample if (1.inexistent_method rescue false)
+ # first
+ # second
+ def a
+ end
+end
+ RUBY
+
+ @parser.scan
+
+ c = @store.find_class_named 'C'
+
+ c_a = c.find_method_named 'a'
+ assert_equal "first\nsecond", c_a.comment.text
+ end
+
def test_singleton_method_via_eigenclass
util_parser <<-RUBY
class C
diff --git a/test/rdoc/test_rdoc_ruby_lex.rb b/test/rdoc/test_rdoc_ruby_lex.rb
deleted file mode 100644
index 9ab5844b54..0000000000
--- a/test/rdoc/test_rdoc_ruby_lex.rb
+++ /dev/null
@@ -1,1095 +0,0 @@
-# coding: UTF-8
-# frozen_string_literal: false
-
-require 'rdoc/test_case'
-
-class TestRDocRubyLex < RDoc::TestCase
-
- def setup
- @TK = RDoc::RubyToken
- end
-
- def test_token_position
- tokens = RDoc::RubyLex.tokenize '[ 1, :a, nil ]', nil
-
- assert_equal '[', tokens[0].text
- assert_equal 0, tokens[0].seek
- assert_equal 1, tokens[0].line_no
- assert_equal 0, tokens[0].char_no
- assert_equal '1', tokens[2].text
- assert_equal 2, tokens[2].seek
- assert_equal 1, tokens[2].line_no
- assert_equal 2, tokens[2].char_no
- assert_equal ':a', tokens[5].text
- assert_equal 5, tokens[5].seek
- assert_equal 1, tokens[5].line_no
- assert_equal 5, tokens[5].char_no
- assert_equal 'nil', tokens[8].text
- assert_equal 9, tokens[8].seek
- assert_equal 1, tokens[8].line_no
- assert_equal 9, tokens[8].char_no
- assert_equal ']', tokens[10].text
- assert_equal 13, tokens[10].seek
- assert_equal 1, tokens[10].line_no
- assert_equal 13, tokens[10].char_no
- end
-
- def test_class_tokenize
- tokens = RDoc::RubyLex.tokenize "def x() end", nil
-
- expected = [
- @TK::TkDEF .new( 0, 1, 0, "def"),
- @TK::TkSPACE .new( 3, 1, 3, " "),
- @TK::TkIDENTIFIER.new( 4, 1, 4, "x"),
- @TK::TkLPAREN .new( 5, 1, 5, "("),
- @TK::TkRPAREN .new( 6, 1, 6, ")"),
- @TK::TkSPACE .new( 7, 1, 7, " "),
- @TK::TkEND .new( 8, 1, 8, "end"),
- @TK::TkNL .new(11, 1, 11, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize___END__
- tokens = RDoc::RubyLex.tokenize '__END__', nil
-
- expected = [
- @TK::TkEND_OF_SCRIPT.new(0, 1, 0, '__END__'),
- @TK::TkNL .new(7, 1, 7, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize___ENCODING__
- tokens = RDoc::RubyLex.tokenize '__ENCODING__', nil
-
- expected = [
- @TK::Tk__ENCODING__.new( 0, 1, 0, '__ENCODING__'),
- @TK::TkNL .new(12, 1, 12, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_character_literal
- tokens = RDoc::RubyLex.tokenize "?c", nil
-
- expected = [
- @TK::TkCHAR.new( 0, 1, 0, "?c"),
- @TK::TkNL .new( 2, 1, 2, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_character_literal_with_escape
- tokens = RDoc::RubyLex.tokenize "?\\s", nil
-
- expected = [
- @TK::TkCHAR.new( 0, 1, 0, "?\\s"),
- @TK::TkNL .new( 3, 1, 3, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_def_heredoc
- tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
-def x
- < "foo" }', nil
-
- expected = [
- @TK::TkLBRACE .new( 0, 1, 0, '{'),
- @TK::TkSPACE .new( 1, 1, 1, ' '),
- @TK::TkSYMBOL .new( 2, 1, 2, ':class'),
- @TK::TkSPACE .new( 8, 1, 8, ' '),
- @TK::TkHASHROCKET.new( 9, 1, 9, '=>'),
- @TK::TkSPACE .new(11, 1, 11, ' '),
- @TK::TkSTRING .new(12, 1, 12, '"foo"'),
- @TK::TkSPACE .new(17, 1, 17, ' '),
- @TK::TkRBRACE .new(18, 1, 18, '}'),
- @TK::TkNL .new(19, 1, 19, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_heredoc_CR_NL
- tokens = RDoc::RubyLex.tokenize <<-RUBY, nil
-string = <<-STRING\r
-Line 1\r
-Line 2\r
- STRING\r
- RUBY
-
- expected = [
- @TK::TkIDENTIFIER.new( 0, 1, 0, 'string'),
- @TK::TkSPACE .new( 6, 1, 6, ' '),
- @TK::TkASSIGN .new( 7, 1, 7, '='),
- @TK::TkSPACE .new( 8, 1, 8, ' '),
- @TK::TkHEREDOCBEG.new( 9, 1, 9, '<<-STRING'),
- @TK::TkSPACE .new(18, 1, 18, "\r"),
- @TK::TkNL .new(19, 1, 19, "\n"),
- @TK::TkHEREDOC .new(19, 1, 19,
- %Q{Line 1\nLine 2\n}),
- @TK::TkHEREDOCEND.new(45, 4, 36, " STRING\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_opassign
- tokens = RDoc::RubyLex.tokenize <<'RUBY', nil
-a %= b
-a /= b
-a -= b
-a += b
-a |= b
-a &= b
-a >>= b
-a <<= b
-a *= b
-a &&= b
-a ||= b
-a **= b
-RUBY
-
- expected = [
- @TK::TkIDENTIFIER.new( 0, 1, 0, "a"),
- @TK::TkSPACE .new( 1, 1, 1, " "),
- @TK::TkOPASGN .new( 2, 1, 2, "%"),
- @TK::TkSPACE .new( 4, 1, 4, " "),
- @TK::TkIDENTIFIER.new( 5, 1, 5, "b"),
- @TK::TkNL .new( 6, 1, 6, "\n"),
- @TK::TkIDENTIFIER.new( 7, 2, 0, "a"),
- @TK::TkSPACE .new( 8, 2, 1, " "),
- @TK::TkOPASGN .new( 9, 2, 2, "/"),
- @TK::TkSPACE .new( 11, 2, 4, " "),
- @TK::TkIDENTIFIER.new( 12, 2, 5, "b"),
- @TK::TkNL .new( 13, 2, 7, "\n"),
- @TK::TkIDENTIFIER.new( 14, 3, 0, "a"),
- @TK::TkSPACE .new( 15, 3, 1, " "),
- @TK::TkOPASGN .new( 16, 3, 2, "-"),
- @TK::TkSPACE .new( 18, 3, 4, " "),
- @TK::TkIDENTIFIER.new( 19, 3, 5, "b"),
- @TK::TkNL .new( 20, 3, 14, "\n"),
- @TK::TkIDENTIFIER.new( 21, 4, 0, "a"),
- @TK::TkSPACE .new( 22, 4, 1, " "),
- @TK::TkOPASGN .new( 23, 4, 2, "+"),
- @TK::TkSPACE .new( 25, 4, 4, " "),
- @TK::TkIDENTIFIER.new( 26, 4, 5, "b"),
- @TK::TkNL .new( 27, 4, 21, "\n"),
- @TK::TkIDENTIFIER.new( 28, 5, 0, "a"),
- @TK::TkSPACE .new( 29, 5, 1, " "),
- @TK::TkOPASGN .new( 30, 5, 2, "|"),
- @TK::TkSPACE .new( 32, 5, 4, " "),
- @TK::TkIDENTIFIER.new( 33, 5, 5, "b"),
- @TK::TkNL .new( 34, 5, 28, "\n"),
- @TK::TkIDENTIFIER.new( 35, 6, 0, "a"),
- @TK::TkSPACE .new( 36, 6, 1, " "),
- @TK::TkOPASGN .new( 37, 6, 2, "&"),
- @TK::TkSPACE .new( 39, 6, 4, " "),
- @TK::TkIDENTIFIER.new( 40, 6, 5, "b"),
- @TK::TkNL .new( 41, 6, 35, "\n"),
- @TK::TkIDENTIFIER.new( 42, 7, 0, "a"),
- @TK::TkSPACE .new( 43, 7, 1, " "),
- @TK::TkOPASGN .new( 44, 7, 2, ">>"),
- @TK::TkSPACE .new( 47, 7, 5, " "),
- @TK::TkIDENTIFIER.new( 48, 7, 6, "b"),
- @TK::TkNL .new( 49, 7, 42, "\n"),
- @TK::TkIDENTIFIER.new( 50, 8, 0, "a"),
- @TK::TkSPACE .new( 51, 8, 1, " "),
- @TK::TkOPASGN .new( 52, 8, 2, "<<"),
- @TK::TkSPACE .new( 55, 8, 5, " "),
- @TK::TkIDENTIFIER.new( 56, 8, 6, "b"),
- @TK::TkNL .new( 57, 8, 50, "\n"),
- @TK::TkIDENTIFIER.new( 58, 9, 0, "a"),
- @TK::TkSPACE .new( 59, 9, 1, " "),
- @TK::TkOPASGN .new( 60, 9, 2, "*"),
- @TK::TkSPACE .new( 62, 9, 4, " "),
- @TK::TkIDENTIFIER.new( 63, 9, 5, "b"),
- @TK::TkNL .new( 64, 9, 58, "\n"),
- @TK::TkIDENTIFIER.new( 65, 10, 0, "a"),
- @TK::TkSPACE .new( 66, 10, 1, " "),
- @TK::TkOPASGN .new( 67, 10, 2, "&&"),
- @TK::TkSPACE .new( 70, 10, 5, " "),
- @TK::TkIDENTIFIER.new( 71, 10, 6, "b"),
- @TK::TkNL .new( 72, 10, 65, "\n"),
- @TK::TkIDENTIFIER.new( 73, 11, 0, "a"),
- @TK::TkSPACE .new( 74, 11, 1, " "),
- @TK::TkOPASGN .new( 75, 11, 2, "||"),
- @TK::TkSPACE .new( 78, 11, 5, " "),
- @TK::TkIDENTIFIER.new( 79, 11, 6, "b"),
- @TK::TkNL .new( 80, 11, 73, "\n"),
- @TK::TkIDENTIFIER.new( 81, 12, 0, "a"),
- @TK::TkSPACE .new( 82, 12, 1, " "),
- @TK::TkOPASGN .new( 83, 12, 2, "**"),
- @TK::TkSPACE .new( 86, 12, 5, " "),
- @TK::TkIDENTIFIER.new( 87, 12, 6, "b"),
- @TK::TkNL .new( 88, 12, 81, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_heredoc_call
- tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
-string = <<-STRING.chomp
-Line 1
-Line 2
- STRING
- RUBY
-
- expected = [
- @TK::TkIDENTIFIER.new( 0, 1, 0, 'string'),
- @TK::TkSPACE .new( 6, 1, 6, ' '),
- @TK::TkASSIGN .new( 7, 1, 7, '='),
- @TK::TkSPACE .new( 8, 1, 8, ' '),
- @TK::TkHEREDOCBEG.new( 9, 1, 9, '<<-STRING'),
- @TK::TkDOT .new(18, 1, 18, '.'),
- @TK::TkIDENTIFIER.new(19, 1, 19, 'chomp'),
- @TK::TkNL .new(24, 1, 24, "\n"),
- @TK::TkHEREDOC .new(24, 1, 24, "Line 1\nLine 2\n"),
- @TK::TkHEREDOCEND.new(47, 4, 39, " STRING\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_heredoc_indent
- tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
-string = <<-STRING
-Line 1
-Line 2
- STRING
- RUBY
-
- expected = [
- @TK::TkIDENTIFIER.new( 0, 1, 0, 'string'),
- @TK::TkSPACE .new( 6, 1, 6, ' '),
- @TK::TkASSIGN .new( 7, 1, 7, '='),
- @TK::TkSPACE .new( 8, 1, 8, ' '),
-
-
- @TK::TkHEREDOCBEG.new( 9, 1, 9, '<<-STRING'),
- @TK::TkNL .new(18, 1, 18, "\n"),
- @TK::TkHEREDOC .new(18, 1, 18, "Line 1\nLine 2\n"),
- @TK::TkHEREDOCEND.new(41, 4, 33, " STRING\n")
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_heredoc_missing_end
- e = assert_raises RDoc::RubyLex::Error do
- RDoc::RubyLex.tokenize <<-'RUBY', nil
->> string1 = <<-TXT
->" That's swell
->" TXT
- RUBY
- end
-
- assert_equal 'Missing terminating TXT for string', e.message
- end
-
- def test_class_tokenize_heredoc_percent_N
- tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
-a b <<-U
-%N
-U
- RUBY
-
- expected = [
- @TK::TkIDENTIFIER.new( 0, 1, 0, 'a'),
- @TK::TkSPACE .new( 1, 1, 1, ' '),
- @TK::TkIDENTIFIER.new( 2, 1, 2, 'b'),
- @TK::TkSPACE .new( 3, 1, 3, ' '),
- @TK::TkHEREDOCBEG.new( 4, 1, 4, '<<-U'),
- @TK::TkNL .new( 8, 1, 8, "\n"),
- @TK::TkHEREDOC .new( 8, 1, 8, "%N\n"),
- @TK::TkHEREDOCEND.new(13, 3, 12, "U\n")
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_identifier_high_unicode
- tokens = RDoc::RubyLex.tokenize '𝖒', nil
-
- expected = @TK::TkIDENTIFIER.new(0, 1, 0, '𝖒')
-
- assert_equal expected, tokens.first
- end
-
- def test_class_tokenize_lambda
- tokens = RDoc::RubyLex.tokenize 'a = -> x, y { x + y }', nil
-
- expected = [
- @TK::TkIDENTIFIER.new( 0, 1, 0, 'a'),
- @TK::TkSPACE .new( 1, 1, 1, ' '),
- @TK::TkASSIGN .new( 2, 1, 2, '='),
- @TK::TkSPACE .new( 3, 1, 3, ' '),
- @TK::TkLAMBDA .new( 4, 1, 4, '->'),
- @TK::TkSPACE .new( 6, 1, 6, ' '),
- @TK::TkIDENTIFIER.new( 7, 1, 7, 'x'),
- @TK::TkCOMMA .new( 8, 1, 8, ','),
- @TK::TkSPACE .new( 9, 1, 9, ' '),
- @TK::TkIDENTIFIER.new(10, 1, 10, 'y'),
- @TK::TkSPACE .new(11, 1, 11, ' '),
- @TK::TkfLBRACE .new(12, 1, 12, '{'),
- @TK::TkSPACE .new(13, 1, 13, ' '),
- @TK::TkIDENTIFIER.new(14, 1, 14, 'x'),
- @TK::TkSPACE .new(15, 1, 15, ' '),
- @TK::TkPLUS .new(16, 1, 16, '+'),
- @TK::TkSPACE .new(17, 1, 17, ' '),
- @TK::TkIDENTIFIER.new(18, 1, 18, 'y'),
- @TK::TkSPACE .new(19, 1, 19, ' '),
- @TK::TkRBRACE .new(20, 1, 20, '}'),
- @TK::TkNL .new(21, 1, 21, "\n")
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_percent_1
- tokens = RDoc::RubyLex.tokenize 'v%10==10', nil
-
- expected = [
- @TK::TkIDENTIFIER.new(0, 1, 0, 'v'),
- @TK::TkMOD.new( 1, 1, 1, '%'),
- @TK::TkINTEGER.new( 2, 1, 2, '10'),
- @TK::TkEQ.new( 4, 1, 4, '=='),
- @TK::TkINTEGER.new( 6, 1, 6, '10'),
- @TK::TkNL.new( 8, 1, 8, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_percent_r
- tokens = RDoc::RubyLex.tokenize '%r[hi]', nil
-
- expected = [
- @TK::TkREGEXP.new( 0, 1, 0, '%r[hi]'),
- @TK::TkNL .new( 6, 1, 6, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_percent_r_with_slash
- tokens = RDoc::RubyLex.tokenize '%r/hi/', nil
-
- expected = [
- @TK::TkREGEXP.new( 0, 1, 0, '%r/hi/'),
- @TK::TkNL .new( 6, 1, 6, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_percent_large_q
- tokens = RDoc::RubyLex.tokenize '%Q/hi/', nil
-
- expected = [
- @TK::TkSTRING.new( 0, 1, 0, '%Q/hi/'),
- @TK::TkNL .new( 6, 1, 6, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_percent_large_q_with_double_quote
- tokens = RDoc::RubyLex.tokenize '%Q"hi"', nil
-
- expected = [
- @TK::TkSTRING.new( 0, 1, 0, '%Q"hi"'),
- @TK::TkNL .new( 6, 1, 6, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_percent_w
- tokens = RDoc::RubyLex.tokenize '%w[hi]', nil
-
- expected = [
- @TK::TkDSTRING.new( 0, 1, 0, '%w[hi]'),
- @TK::TkNL .new( 6, 1, 6, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_percent_w_quote
- tokens = RDoc::RubyLex.tokenize '%w"hi"', nil
-
- expected = [
- @TK::TkDSTRING.new( 0, 1, 0, '%w"hi"'),
- @TK::TkNL .new( 6, 1, 6, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_hash_rocket
- tokens = RDoc::RubyLex.tokenize "{ :foo=> 1 }", nil
-
- expected = [
- @TK::TkLBRACE .new( 0, 1, 0, '{'),
- @TK::TkSPACE .new( 1, 1, 1, ' '),
- @TK::TkSYMBOL .new( 2, 1, 2, ':foo'),
- @TK::TkHASHROCKET.new( 6, 1, 6, '=>'),
- @TK::TkSPACE .new( 8, 1, 8, ' '),
- @TK::TkINTEGER .new( 9, 1, 9, '1'),
- @TK::TkSPACE .new(10, 1, 10, ' '),
- @TK::TkRBRACE .new(11, 1, 11, '}'),
- @TK::TkNL .new(12, 1, 12, "\n")
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_percent_sign_quote
- tokens = RDoc::RubyLex.tokenize '%%hi%', nil
-
- expected = [
- @TK::TkSTRING.new( 0, 1, 0, '%%hi%'),
- @TK::TkNL .new( 5, 1, 5, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_regexp
- tokens = RDoc::RubyLex.tokenize "/hay/", nil
-
- expected = [
- @TK::TkREGEXP.new( 0, 1, 0, "/hay/"),
- @TK::TkNL .new( 5, 1, 5, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_regexp_options
- tokens = RDoc::RubyLex.tokenize "/hAY/i", nil
-
- expected = [
- @TK::TkREGEXP.new( 0, 1, 0, "/hAY/i"),
- @TK::TkNL .new( 6, 1, 6, "\n"),
- ]
-
- assert_equal expected, tokens
-
- tokens = RDoc::RubyLex.tokenize "/hAY/ix", nil
-
- expected = [
- @TK::TkREGEXP.new( 0, 1, 0, "/hAY/ix"),
- @TK::TkNL .new( 7, 1, 7, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_regexp_backref
- tokens = RDoc::RubyLex.tokenize "/[csh](..) [csh]\\1 in/", nil
-
- expected = [
- @TK::TkREGEXP.new( 0, 1, 0, "/[csh](..) [csh]\\1 in/"),
- @TK::TkNL .new(22, 1, 22, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_regexp_escape
- tokens = RDoc::RubyLex.tokenize "/\\//", nil
-
- expected = [
- @TK::TkREGEXP.new( 0, 1, 0, "/\\//"),
- @TK::TkNL .new( 4, 1, 4, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_number_with_sign_character
- tokens = RDoc::RubyLex.tokenize "+3--3r", nil
-
- expected = [
- @TK::TkINTEGER .new(0, 1, 0, "+3"),
- @TK::TkMINUS .new(2, 1, 2, "-"),
- @TK::TkRATIONAL.new(3, 1, 3, "-3r"),
- @TK::TkNL .new(6, 1, 6, "\n"),
- ]
-
- assert_equal expected, tokens
- end
-
- def test_class_tokenize_regexp_continuing_backslash
- tokens = RDoc::RubyLex.tokenize "/(? 0, :char_no => 0, :kind => :on_const, :text => 'CONSTANT' },
+ { :line_no => 0, :char_no => 0, :kind => :on_kw, :text => 'KW' },
+ { :line_no => 0, :char_no => 0, :kind => :on_ivar, :text => 'IVAR' },
+ { :line_no => 0, :char_no => 0, :kind => :on_op, :text => 'Op' },
+ { :line_no => 0, :char_no => 0, :kind => :on_ident, :text => 'Id' },
+ { :line_no => 0, :char_no => 0, :kind => :on_backref, :text => 'Node' },
+ { :line_no => 0, :char_no => 0, :kind => :on_comment, :text => 'COMMENT' },
+ { :line_no => 0, :char_no => 0, :kind => :on_regexp, :text => 'REGEXP' },
+ { :line_no => 0, :char_no => 0, :kind => :on_tstring, :text => 'STRING' },
+ { :line_no => 0, :char_no => 0, :kind => :on_int, :text => 'Val' },
+ { :line_no => 0, :char_no => 0, :kind => :on_unknown, :text => '\\' }
]
expected = [