Merge rdoc-6.0.0.beta1.

This version fixed strange behavior of ruby code parser.
  We will list all of impromovement to Changelog when 6.0.0 releasing.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@59686 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

lib/rdoc.rb

@@ -65,7 +65,7 @@ module RDoc
   ##
   # RDoc version you are using
 
-  VERSION = '5.1.0'
+  VERSION = '6.0.0.beta1'
 
   ##
   # Method visibilities

lib/rdoc/code_object.rb

@@ -69,13 +69,6 @@ class RDoc::CodeObject
 
   attr_reader :metadata
 
-  ##
-  # Offset in #file where this CodeObject was defined
-  #--
-  # TODO character or byte?
-
-  attr_accessor :offset
-
   ##
   # Sets the parent CodeObject
 

lib/rdoc/context.rb

@@ -762,7 +762,7 @@ class RDoc::Context < RDoc::CodeObject
     attributes.default = []
 
     sort_sections.each do |section|
-      yield section, constants[section].sort, attributes[section].sort
+      yield section, constants[section].select(&:display?).sort, attributes[section].select(&:display?).sort
     end
   end
 

lib/rdoc/generator/template/darkfish/class.rhtml

@@ -26,8 +26,6 @@
   </section>
 
   <% klass.each_section do |section, constants, attributes| %>
-  <% constants = constants.select { |const| const.display? } %>
-  <% attributes = attributes.select { |attr| attr.display? } %>
   <section id="<%= section.aref %>" class="documentation-section">
     <% if section.title then %>
     <header class="documentation-section-title">

lib/rdoc/i18n.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: false
 ##
-# This module provides i18n related features.
+# This module provides i18n realated features.
 
 module RDoc::I18n
 

lib/rdoc/markdown.rb

@@ -15280,7 +15280,7 @@ class RDoc::Markdown
         self.pos = _save
         break
       end
-      @result = begin;
+      @result = begin; 
                ref = [:inline, @note_order.length]
                @footnotes[ref] = paragraph a
 

lib/rdoc/parser/c.rb

@@ -670,7 +670,6 @@ class RDoc::Parser::C < RDoc::Parser
       tk.set_text body
       meth_obj.add_token tk
       meth_obj.comment = comment
-      meth_obj.offset  = offset
       meth_obj.line    = file_content[0, offset].count("\n") + 1
 
       body
@@ -689,7 +688,6 @@ class RDoc::Parser::C < RDoc::Parser
       tk.set_text body
       meth_obj.add_token tk
       meth_obj.comment = comment
-      meth_obj.offset  = offset
       meth_obj.line    = file_content[0, offset].count("\n") + 1
 
       body

lib/rdoc/parser/ruby.rb

@@ -589,7 +589,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # +comment+.
 
   def parse_attr(context, single, tk, comment)
-    offset  = tk.seek
     line_no = tk.line_no
 
     args = parse_symbol_arg 1
@@ -606,7 +605,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
       end
 
       att = create_attr context, single, name, rw, comment
-      att.offset = offset
       att.line   = line_no
 
       read_documentation_modifiers att, RDoc::ATTR_MODIFIERS
@@ -620,7 +618,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # comment for each to +comment+.
 
   def parse_attr_accessor(context, single, tk, comment)
-    offset  = tk.seek
     line_no = tk.line_no
 
     args = parse_symbol_arg
@@ -642,7 +639,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     for name in args
       att = create_attr context, single, name, rw, comment
-      att.offset = offset
       att.line   = line_no
     end
   end
@@ -651,7 +647,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Parses an +alias+ in +context+ with +comment+
 
   def parse_alias(context, single, tk, comment)
-    offset  = tk.seek
     line_no = tk.line_no
 
     skip_tkspace
@@ -680,7 +675,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
     al = RDoc::Alias.new(get_tkread, old_name, new_name, comment,
                          single == SINGLE)
     record_location al
-    al.offset = offset
     al.line   = line_no
 
     read_documentation_modifiers al, RDoc::ATTR_MODIFIERS
@@ -733,7 +727,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Parses a class in +context+ with +comment+
 
   def parse_class container, single, tk, comment
-    offset  = tk.seek
     line_no = tk.line_no
 
     declaration_context = container
@@ -748,7 +741,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
         case name = get_class_specification
         when 'self', container.name
           parse_statements container, SINGLE
-          return # don't update offset or line
+          return # don't update line
         else
           parse_class_singleton container, name, comment
         end
@@ -757,9 +750,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
         return
       end
 
-    cls.offset = offset
     cls.line   = line_no
 
+    # after end modifiers
+    read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
+
     cls
   end
 
@@ -845,7 +840,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # true, no found constants will be added to RDoc.
 
   def parse_constant container, tk, comment, ignore_constants = false
-    offset  = tk.seek
     line_no = tk.line_no
 
     name = tk.name
@@ -885,7 +879,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     value.replace body
     record_location con
-    con.offset = offset
     con.line   = line_no
     read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS
 
@@ -950,7 +943,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   def parse_comment container, tk, comment
     return parse_comment_tomdoc container, tk, comment if @markup == 'tomdoc'
     column  = tk.char_no
-    offset  = tk.seek
     line_no = tk.line_no
 
     text = comment.text
@@ -966,7 +958,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     if co then
       co.singleton = singleton
-      co.offset    = offset
       co.line      = line_no
     end
 
@@ -1031,19 +1022,18 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def parse_comment_tomdoc container, tk, comment
     return unless signature = RDoc::TomDoc.signature(comment)
-    offset  = tk.seek
+    column  = tk.char_no
     line_no = tk.line_no
 
     name, = signature.split %r%[ \(]%, 2
 
     meth = RDoc::GhostMethod.new get_tkread, name
     record_location meth
-    meth.offset    = offset
     meth.line      = line_no
 
     meth.start_collecting_tokens
     indent = TkSPACE.new 0, 1, 1
-    indent.set_text " " * offset
+    indent.set_text " " * column
 
     position_comment = TkCOMMENT.new 0, line_no, 1
     position_comment.set_text "# File #{@top_level.relative_name}, line #{line_no}"
@@ -1183,7 +1173,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def parse_meta_method(container, single, tk, comment)
     column  = tk.char_no
-    offset  = tk.seek
     line_no = tk.line_no
 
     start_collecting_tokens
@@ -1200,7 +1189,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     meth = RDoc::MetaMethod.new get_tkread, name
     record_location meth
-    meth.offset = offset
     meth.line   = line_no
     meth.singleton = singleton
 
@@ -1291,7 +1279,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
     added_container = false
     name = nil
     column  = tk.char_no
-    offset  = tk.seek
     line_no = tk.line_no
 
     start_collecting_tokens
@@ -1309,7 +1296,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
     meth.singleton = single == SINGLE ? true : singleton
 
     record_location meth
-    meth.offset = offset
     meth.line   = line_no
 
     meth.start_collecting_tokens
@@ -1328,6 +1314,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     meth.comment = comment
 
+    # after end modifiers
+    read_documentation_modifiers meth, RDoc::METHOD_MODIFIERS
+
     @stats.add_method meth
   end
 
@@ -1519,6 +1508,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       end
       tk = get_tk
     end
+    @scanner.first_in_method_statement = true
 
     get_tkread_clean(/\s+/, ' ')
   end
@@ -1559,6 +1549,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
     mod.add_comment comment, @top_level
     parse_statements mod
 
+    # after end modifiers
+    read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS
+
     @stats.add_module mod
   end
 
@@ -1732,7 +1725,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
       when TkEND then
         nest -= 1
         if nest == 0 then
-          read_documentation_modifiers container, RDoc::CLASS_MODIFIERS
           container.ongoing_visibility = save_visibility
 
           parse_comment container, tk, comment unless comment.empty?
@@ -2039,7 +2031,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   def skip_optional_do_after_expression
     skip_tkspace false
     tk = get_tk
-    end_token = get_end_token tk
 
     b_nest = 0
     nest = 0
@@ -2047,23 +2038,18 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     loop do
       case tk
-      when TkSEMICOLON then
+      when TkSEMICOLON, TkNL then
         break if b_nest.zero?
       when TkLPAREN, TkfLPAREN then
         nest += 1
+      when TkRPAREN then
+        nest -= 1
       when TkBEGIN then
         b_nest += 1
       when TkEND then
         b_nest -= 1
       when TkDO
         break if nest.zero?
-      when end_token then
-        if end_token == TkRPAREN
-          nest -= 1
-          break if @scanner.lex_state == :EXPR_END and nest.zero?
-        else
-          break unless @scanner.continue
-        end
       when nil then
         break
       end

lib/rdoc/parser/ruby_tools.rb

@@ -33,26 +33,6 @@ module RDoc::Parser::RubyTools
 
     tk = nil if TkEND_OF_SCRIPT === tk
 
-    if TkSYMBEG === tk then
-      set_token_position tk.line_no, tk.char_no
-
-      case tk1 = get_tk
-      when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then
-        if tk1.respond_to?(:name) then
-          tk = Token(TkSYMBOL).set_text(":" + tk1.name)
-        else
-          tk = Token(TkSYMBOL).set_text(":" + tk1.text)
-        end
-
-        # remove the identifier we just read to replace it with a symbol
-        @token_listeners.each do |obj|
-          obj.pop_token
-        end if @token_listeners
-      else
-        tk = tk1
-      end
-    end
-
     # inform any listeners of our shiny new token
     @token_listeners.each do |obj|
       obj.add_token(tk)

lib/rdoc/rd/block_parser.rb

@@ -1,7 +1,7 @@
 #
 # DO NOT MODIFY!!!!
 # This file is automatically generated by Racc 1.4.14
-# from Racc grammar file "".
+# from Racc grammer file "".
 #
 
 require 'racc/parser.rb'
@@ -677,54 +677,54 @@ Racc_debug_parser = false
 # reduce 0 omitted
 
 def _reduce_1(val, _values, result)
- result = RDoc::Markup::Document.new(*val[0])
+ result = RDoc::Markup::Document.new(*val[0]) 
     result
 end
 
 def _reduce_2(val, _values, result)
- raise ParseError, "file empty"
+ raise ParseError, "file empty" 
     result
 end
 
 def _reduce_3(val, _values, result)
- result = val[0].concat val[1]
+ result = val[0].concat val[1] 
     result
 end
 
 def _reduce_4(val, _values, result)
- result = val[0]
+ result = val[0] 
     result
 end
 
 def _reduce_5(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_6(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 # reduce 7 omitted
 
 def _reduce_8(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_9(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_10(val, _values, result)
- result = [RDoc::Markup::BlankLine.new]
+ result = [RDoc::Markup::BlankLine.new] 
     result
 end
 
 def _reduce_11(val, _values, result)
- result = val[0].parts
+ result = val[0].parts 
     result
 end
 
@@ -732,30 +732,30 @@ def _reduce_12(val, _values, result)
       # val[0] is like [level, title]
       title = @inline_parser.parse(val[0][1])
       result = RDoc::Markup::Heading.new(val[0][0], title)
-
+    
     result
 end
 
 def _reduce_13(val, _values, result)
       result = RDoc::Markup::Include.new val[0], @include_path
-
+    
     result
 end
 
 def _reduce_14(val, _values, result)
       # val[0] is Array of String
       result = paragraph val[0]
-
+    
     result
 end
 
 def _reduce_15(val, _values, result)
- result << val[1].rstrip
+ result << val[1].rstrip 
     result
 end
 
 def _reduce_16(val, _values, result)
- result = [val[0].rstrip]
+ result = [val[0].rstrip] 
     result
 end
 
@@ -766,7 +766,7 @@ def _reduce_17(val, _values, result)
 
       # imform to lexer.
       @in_verbatim = false
-
+    
     result
 end
 
@@ -777,25 +777,25 @@ def _reduce_18(val, _values, result)
 
       # imform to lexer.
       @in_verbatim = false
-
+    
     result
 end
 
 def _reduce_19(val, _values, result)
       result << val[1]
-
+    
     result
 end
 
 def _reduce_20(val, _values, result)
       result.concat val[2]
-
+    
     result
 end
 
 def _reduce_21(val, _values, result)
       result << "\n"
-
+    
     result
 end
 
@@ -803,7 +803,7 @@ def _reduce_22(val, _values, result)
       result = val
       # inform to lexer.
       @in_verbatim = true
-
+    
     result
 end
 
@@ -817,89 +817,89 @@ end
 
 def _reduce_27(val, _values, result)
       result = val[0]
-
+    
     result
 end
 
 def _reduce_28(val, _values, result)
       result = val[1]
-
+    
     result
 end
 
 def _reduce_29(val, _values, result)
       result = val[1].push(val[2])
-
+    
     result
 end
 
 def _reduce_30(val, _values, result)
- result = val[0] << val[1]
+ result = val[0] << val[1] 
     result
 end
 
 def _reduce_31(val, _values, result)
- result = [val[0]]
+ result = [val[0]] 
     result
 end
 
 def _reduce_32(val, _values, result)
       result = RDoc::Markup::List.new :BULLET, *val[0]
-
+    
     result
 end
 
 def _reduce_33(val, _values, result)
- result.push(val[1])
+ result.push(val[1]) 
     result
 end
 
 def _reduce_34(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_35(val, _values, result)
       result = RDoc::Markup::ListItem.new nil, val[0], *val[1]
-
+    
     result
 end
 
 def _reduce_36(val, _values, result)
       result = RDoc::Markup::List.new :NUMBER, *val[0]
-
+    
     result
 end
 
 def _reduce_37(val, _values, result)
- result.push(val[1])
+ result.push(val[1]) 
     result
 end
 
 def _reduce_38(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_39(val, _values, result)
       result = RDoc::Markup::ListItem.new nil, val[0], *val[1]
-
+    
     result
 end
 
 def _reduce_40(val, _values, result)
       result = RDoc::Markup::List.new :NOTE, *val[0]
-
+    
     result
 end
 
 def _reduce_41(val, _values, result)
- result.push(val[1])
+ result.push(val[1]) 
     result
 end
 
 def _reduce_42(val, _values, result)
- result = val
+ result = val 
     result
 end
 
@@ -907,77 +907,77 @@ def _reduce_43(val, _values, result)
       term = @inline_parser.parse val[0].strip
 
       result = RDoc::Markup::ListItem.new term, *val[1]
-
+    
     result
 end
 
 def _reduce_44(val, _values, result)
       result = RDoc::Markup::List.new :LABEL, *val[0]
-
+    
     result
 end
 
 def _reduce_45(val, _values, result)
- result.push(val[1])
+ result.push(val[1]) 
     result
 end
 
 def _reduce_46(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_47(val, _values, result)
       result = RDoc::Markup::ListItem.new "<tt>#{val[0].strip}</tt>", *val[1]
-
+    
     result
 end
 
 def _reduce_48(val, _values, result)
       result = [val[1]].concat(val[2])
-
+    
     result
 end
 
 def _reduce_49(val, _values, result)
       result = [val[1]]
-
+    
     result
 end
 
 def _reduce_50(val, _values, result)
       result = val[2]
-
+    
     result
 end
 
 def _reduce_51(val, _values, result)
       result = []
-
+    
     result
 end
 
 def _reduce_52(val, _values, result)
- result.concat val[1]
+ result.concat val[1] 
     result
 end
 
 # reduce 53 omitted
 
 def _reduce_54(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_55(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 # reduce 56 omitted
 
 def _reduce_57(val, _values, result)
- result = []
+ result = [] 
     result
 end
 
@@ -991,58 +991,58 @@ end
 
 def _reduce_62(val, _values, result)
       result = paragraph [val[0]].concat(val[1])
-
+    
     result
 end
 
 def _reduce_63(val, _values, result)
       result = paragraph [val[0]]
-
+    
     result
 end
 
 def _reduce_64(val, _values, result)
       result = paragraph [val[0]].concat(val[1])
-
+    
     result
 end
 
 def _reduce_65(val, _values, result)
       result = paragraph [val[0]]
-
+    
     result
 end
 
 def _reduce_66(val, _values, result)
       result = [val[0]].concat(val[1])
-
+    
     result
 end
 
 def _reduce_67(val, _values, result)
- result.concat val[1]
+ result.concat val[1] 
     result
 end
 
 def _reduce_68(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
 def _reduce_69(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 # reduce 70 omitted
 
 def _reduce_71(val, _values, result)
- result = []
+ result = [] 
     result
 end
 
 def _reduce_72(val, _values, result)
- result = []
+ result = [] 
     result
 end
 

lib/rdoc/rd/inline_parser.rb

@@ -1,7 +1,7 @@
 #
 # DO NOT MODIFY!!!!
 # This file is automatically generated by Racc 1.4.14
-# from Racc grammar file "".
+# from Racc grammer file "".
 #
 
 require 'racc/parser.rb'
@@ -704,9 +704,9 @@ Racc_token_to_s_table = [
   "ref_subst_strings_q",
   "ref_subst_strings_first",
   "ref_subst_ele2",
-  "ref_subst_eels",
+  "ref_subst_eles",
   "ref_subst_str_ele_first",
-  "ref_subst_eels_q",
+  "ref_subst_eles_q",
   "ref_subst_ele",
   "ref_subst_ele_q",
   "ref_subst_str_ele",
@@ -732,12 +732,12 @@ Racc_debug_parser = false
 # reduce 1 omitted
 
 def _reduce_2(val, _values, result)
- result.append val[1]
+ result.append val[1] 
     result
 end
 
 def _reduce_3(val, _values, result)
- result = val[0]
+ result = val[0] 
     result
 end
 
@@ -762,28 +762,28 @@ end
 def _reduce_13(val, _values, result)
       content = val[1]
       result = inline "<em>#{content}</em>", content
-
+    
     result
 end
 
 def _reduce_14(val, _values, result)
       content = val[1]
       result = inline "<code>#{content}</code>", content
-
+    
     result
 end
 
 def _reduce_15(val, _values, result)
       content = val[1]
       result = inline "+#{content}+", content
-
+    
     result
 end
 
 def _reduce_16(val, _values, result)
       content = val[1]
       result = inline "<tt>#{content}</tt>", content
-
+    
     result
 end
 
@@ -791,13 +791,13 @@ def _reduce_17(val, _values, result)
       label = val[1]
       @block_parser.add_label label.reference
       result = "<span id=\"label-#{label}\">#{label}</span>"
-
+    
     result
 end
 
 def _reduce_18(val, _values, result)
       result = "{#{val[1]}}[#{val[2].join}]"
-
+    
     result
 end
 
@@ -805,13 +805,13 @@ def _reduce_19(val, _values, result)
       scheme, inline = val[1]
 
       result = "{#{inline}}[#{scheme}#{inline.reference}]"
-
+    
     result
 end
 
 def _reduce_20(val, _values, result)
       result = [nil, inline(val[1])]
-
+    
     result
 end
 
@@ -820,25 +820,25 @@ def _reduce_21(val, _values, result)
         'rdoc-label:',
         inline("#{val[0].reference}/#{val[1].reference}")
       ]
-
+    
     result
 end
 
 def _reduce_22(val, _values, result)
       result = ['rdoc-label:', val[0].reference]
-
+    
     result
 end
 
 def _reduce_23(val, _values, result)
       result = ['rdoc-label:', "#{val[0].reference}/"]
-
+    
     result
 end
 
 def _reduce_24(val, _values, result)
       result = [nil, inline(val[1])]
-
+    
     result
 end
 
@@ -847,92 +847,92 @@ def _reduce_25(val, _values, result)
         'rdoc-label:',
         inline("#{val[0].reference}/#{val[1].reference}")
       ]
-
+    
     result
 end
 
 def _reduce_26(val, _values, result)
       result = ['rdoc-label:', val[0]]
-
+    
     result
 end
 
 def _reduce_27(val, _values, result)
       ref = val[0].reference
       result = ['rdoc-label:', inline(ref, "#{ref}/")]
-
+    
     result
 end
 
 # reduce 28 omitted
 
 def _reduce_29(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
 def _reduce_30(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
 def _reduce_31(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
 def _reduce_32(val, _values, result)
       result = inline "\"#{val[1]}\""
-
+    
     result
 end
 
 def _reduce_33(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
 def _reduce_34(val, _values, result)
       result = inline "\"#{val[1]}\""
-
+    
     result
 end
 
 # reduce 35 omitted
 
 def _reduce_36(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
 def _reduce_37(val, _values, result)
- result = inline val[1]
+ result = inline val[1] 
     result
 end
 
 def _reduce_38(val, _values, result)
       result = val[0].append val[1]
-
+    
     result
 end
 
 def _reduce_39(val, _values, result)
       result = val[0].append val[1]
-
+    
     result
 end
 
 def _reduce_40(val, _values, result)
       result = val[0]
-
+    
     result
 end
 
 def _reduce_41(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
@@ -940,25 +940,25 @@ end
 
 def _reduce_43(val, _values, result)
       result = val[0].append val[1]
-
+    
     result
 end
 
 def _reduce_44(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
 def _reduce_45(val, _values, result)
       result = val[0].append val[1]
-
+    
     result
 end
 
 def _reduce_46(val, _values, result)
       result = val[0]
-
+    
     result
 end
 
@@ -984,24 +984,24 @@ end
 
 def _reduce_57(val, _values, result)
       result = val[0]
-
+    
     result
 end
 
 def _reduce_58(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
 def _reduce_59(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
 def _reduce_60(val, _values, result)
- result << val[1]
+ result << val[1] 
     result
 end
 
@@ -1009,7 +1009,7 @@ end
 
 def _reduce_62(val, _values, result)
       result << val[1]
-
+    
     result
 end
 
@@ -1017,7 +1017,7 @@ end
 
 def _reduce_64(val, _values, result)
       result << val[1]
-
+    
     result
 end
 
@@ -1048,7 +1048,7 @@ end
 # reduce 77 omitted
 
 def _reduce_78(val, _values, result)
- result << val[1]
+ result << val[1] 
     result
 end
 
@@ -1099,13 +1099,13 @@ end
 def _reduce_101(val, _values, result)
       index = @block_parser.add_footnote val[1].rdoc
       result = "{*#{index}}[rdoc-label:foottext-#{index}:footmark-#{index}]"
-
+    
     result
 end
 
 def _reduce_102(val, _values, result)
       result = inline "<tt>#{val[1]}</tt>", val[1]
-
+    
     result
 end
 
@@ -1122,7 +1122,7 @@ end
 # reduce 108 omitted
 
 def _reduce_109(val, _values, result)
- result << val[1]
+ result << val[1] 
     result
 end
 
@@ -1130,24 +1130,24 @@ end
 
 def _reduce_111(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
 # reduce 112 omitted
 
 def _reduce_113(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
 def _reduce_114(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
 def _reduce_115(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
@@ -1192,7 +1192,7 @@ end
 # reduce 135 omitted
 
 def _reduce_136(val, _values, result)
- result << val[1]
+ result << val[1] 
     result
 end
 

lib/rdoc/rdoc.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
 RDoc produces HTML and command-line documentation for Ruby projects.
 RDoc includes the +rdoc+ and +ri+ tools for generating and displaying documentation from the command-line.
   DESCRIPTION
-  s.homepage = "https://rdoc.github.io/rdoc"
+  s.homepage = "https://ruby.github.io/rdoc"
   s.licenses = ["Ruby"]
 
   s.bindir = "exe"
@@ -46,7 +46,7 @@ RDoc includes the +rdoc+ and +ri+ tools for generating and displaying documentat
     TODO.rdoc
   ]
 
-  s.required_ruby_version = Gem::Requirement.new(">= 1.9.3")
+  s.required_ruby_version = Gem::Requirement.new(">= 2.2.2")
   s.rubygems_version = "2.5.2"
   s.required_rubygems_version = Gem::Requirement.new(">= 2.2")
 

lib/rdoc/ri/driver.rb

@@ -80,7 +80,6 @@ class RDoc::RI::Driver
     options[:interactive] = false
     options[:profile]     = false
     options[:show_all]    = false
-    options[:use_cache]   = true
     options[:use_stdout]  = !$stdout.tty?
     options[:width]       = 72
 
@@ -122,7 +121,7 @@ class RDoc::RI::Driver
       opt.summary_indent = ' ' * 4
 
       opt.banner = <<-EOT
-Usage: #{opt.program_name} [options] [names...]
+Usage: #{opt.program_name} [options] [name ...]
 
 Where name can be:
 
@@ -132,8 +131,8 @@ Where name can be:
 
   gem_name: | gem_name:README | gem_name:History
 
-All class names may be abbreviated to their minimum unambiguous form. If a name
+All class names may be abbreviated to their minimum unambiguous form.
-is ambiguous, all valid options will be listed.
+If a name is ambiguous, all valid options will be listed.
 
 A '.' matches either class or instance methods, while #method
 matches only instance and ::method matches only class methods.
@@ -151,23 +150,23 @@ For example:
     #{opt.program_name} zip
     #{opt.program_name} rdoc:README
 
-Note that shell quoting or escaping may be required for method names containing
+Note that shell quoting or escaping may be required for method names
-punctuation:
+containing punctuation:
 
     #{opt.program_name} 'Array.[]'
     #{opt.program_name} compact\\!
 
-To see the default directories ri will search, run:
+To see the default directories #{opt.program_name} will search, run:
 
     #{opt.program_name} --list-doc-dirs
 
-Specifying the --system, --site, --home, --gems or --doc-dir options will
+Specifying the --system, --site, --home, --gems, or --doc-dir options
-limit ri to searching only the specified directories.
+will limit ri to searching only the specified directories.
 
-ri options may be set in the 'RI' environment variable.
+ri options may be set in the RI environment variable.
 
-The ri pager can be set with the 'RI_PAGER' environment variable or the
+The ri pager can be set with the RI_PAGER environment variable
-'PAGER' environment variable.
+or the PAGER environment variable.
       EOT
 
       opt.separator nil
@@ -199,15 +198,15 @@ The ri pager can be set with the 'RI_PAGER' environment variable or the
       opt.separator nil
 
       opt.on("--[no-]pager",
-             "Send output directly to stdout,",
+             "Send output to a pager,",
-             "rather than to a pager.") do |use_pager|
+             "rather than directly to stdout.") do |use_pager|
         options[:use_stdout] = !use_pager
       end
 
       opt.separator nil
 
       opt.on("-T",
-             "Synonym for --no-pager") do
+             "Synonym for --no-pager.") do
         options[:use_stdout] = true
       end
 
@@ -220,7 +219,7 @@ The ri pager can be set with the 'RI_PAGER' environment variable or the
 
       opt.separator nil
 
-      opt.on("--server [PORT]", Integer,
+      opt.on("--server[=PORT]", Integer,
              "Run RDoc server on the given port.",
              "The default port is 8214.") do |port|
         options[:server] = port || 8214
@@ -235,13 +234,29 @@ The ri pager can be set with the 'RI_PAGER' environment variable or the
       formatters -= %w[html label test] # remove useless output formats
 
       opt.on("--format=NAME", "-f",
-             "Uses the selected formatter. The default",
+             "Use the selected formatter.  The default",
              "formatter is bs for paged output and ansi",
-             "otherwise. Valid formatters are:",
+             "otherwise.  Valid formatters are:",
-             formatters.join(' '), formatters) do |value|
+             "#{formatters.join(', ')}.", formatters) do |value|
         options[:formatter] = RDoc::Markup.const_get "To#{value.capitalize}"
       end
 
+      opt.separator nil
+
+      opt.on("--help", "-h",
+             "Show help and exit.") do
+        puts opts
+        exit
+      end
+
+      opt.separator nil
+
+      opt.on("--version", "-v",
+             "Output version information and exit.") do
+        puts "#{opts.program_name} #{opts.version}"
+        exit
+      end
+
       opt.separator nil
       opt.separator "Data source options:"
       opt.separator nil
@@ -273,7 +288,7 @@ The ri pager can be set with the 'RI_PAGER' environment variable or the
              "Do not include documentation from",
              "the Ruby standard library, site_lib,",
              "installed gems, or ~/.rdoc.",
-             "Use with --doc-dir") do
+             "Use with --doc-dir.") do
         options[:use_system] = false
         options[:use_site] = false
         options[:use_gems] = false
@@ -283,8 +298,8 @@ The ri pager can be set with the 'RI_PAGER' environment variable or the
       opt.separator nil
 
       opt.on("--[no-]system",
-             "Include documentation from Ruby's standard",
+             "Include documentation from Ruby's",
-             "library.  Defaults to true.") do |value|
+             "standard library.  Defaults to true.") do |value|
         options[:use_system] = value
       end
 
@@ -318,14 +333,14 @@ The ri pager can be set with the 'RI_PAGER' environment variable or the
       opt.separator nil
 
       opt.on("--[no-]profile",
-             "Run with the ruby profiler") do |value|
+             "Run with the ruby profiler.") do |value|
         options[:profile] = value
       end
 
       opt.separator nil
 
       opt.on("--dump=CACHE", File,
-             "Dumps data from an ri cache or data file") do |value|
+             "Dump data from an ri cache or data file.") do |value|
         options[:dump_path] = value
       end
     end

lib/rdoc/ruby_lex.rb

@@ -45,6 +45,7 @@ class RDoc::RubyLex
 
   attr_accessor :continue
   attr_accessor :lex_state
+  attr_accessor :first_in_method_statement
   attr_reader :reader
 
   class << self
@@ -106,10 +107,15 @@ class RDoc::RubyLex
     @rests = []
     @seek = 0
 
+    @heredoc_queue = []
+
     @indent = 0
     @indent_stack = []
     @lex_state = :EXPR_BEG
     @space_seen = false
+    @escaped_nl = false
+    @first_in_method_statement = false
+    @after_question = false
 
     @continue = false
     @line = ""
@@ -350,6 +356,7 @@ class RDoc::RubyLex
       begin
         tk = @OP.match(self)
         @space_seen = tk.kind_of?(TkSPACE)
+        @first_in_method_statement = false if !@space_seen && @first_in_method_statement
       rescue SyntaxError => e
         raise Error, "syntax error: #{e.message}" if
           @exception_on_syntax_error
@@ -361,6 +368,28 @@ class RDoc::RubyLex
     if @readed_auto_clean_up
       get_readed
     end
+
+    if TkSYMBEG === tk then
+      tk1 = token
+      set_token_position tk.seek, tk.line_no, tk.char_no
+
+      case tk1
+      when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then
+        if tk1.respond_to?(:name) then
+          tk = Token(TkSYMBOL, ":" + tk1.name)
+        else
+          tk = Token(TkSYMBOL, ":" + tk1.text)
+        end
+      else
+        tk = tk1
+      end
+    elsif (TkPLUS === tk or TkMINUS === tk) and peek(0) =~ /\d/ then
+      tk1 = token
+      set_token_position tk.seek, tk.line_no, tk.char_no
+      tk = Token(tk1.class, tk.text + tk1.text)
+    end
+    @after_question = false if @after_question and !(TkQUESTION === tk)
+
     #      Tracer.off
     tk
   end
@@ -380,7 +409,9 @@ class RDoc::RubyLex
     "r" => "/",
     "w" => "]",
     "W" => "]",
-    "s" => ":"
+    "s" => ":",
+    "i" => "]",
+    "I" => "]"
   }
 
   PERCENT_PAREN = {
@@ -430,15 +461,18 @@ class RDoc::RubyLex
                  proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do
       |op, io|
       @ltype = "="
-      res = ''
+      res = op
-      nil until getc == "\n"
+      until (ch = getc) == "\n" do
+        res << ch
+      end
+      res << ch
 
       until ( peek_equal?("=end") && peek(4) =~ /\s/ ) do
         (ch = getc)
         res << ch
       end
 
-      gets # consume =end
+      res << gets # consume =end
 
       @ltype = nil
       Token(TkRD_COMMENT, res)
@@ -446,28 +480,47 @@ class RDoc::RubyLex
 
     @OP.def_rule("\n") do |op, io|
       print "\\n\n" if RDoc::RubyLex.debug?
-      case @lex_state
+      unless @heredoc_queue.empty?
-      when :EXPR_BEG, :EXPR_FNAME, :EXPR_DOT
+        info = @heredoc_queue[0]
-        @continue = true
+        if !info[:started] # "\n"
-      else
+          info[:started] = true
-        @continue = false
+          ungetc "\n"
-        @lex_state = :EXPR_BEG
+        elsif info[:heredoc_end].nil? # heredoc body
-        until (@indent_stack.empty? ||
+          tk, heredoc_end = identify_here_document_body(info[:quoted], info[:lt], info[:indent])
-               [TkLPAREN, TkLBRACK, TkLBRACE,
+          info[:heredoc_end] = heredoc_end
-                 TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
+          ungetc "\n"
-          @indent_stack.pop
+        else # heredoc end
+          @heredoc_queue.shift
+          @lex_state = :EXPR_BEG
+          tk = Token(TkHEREDOCEND, info[:heredoc_end])
+          if !@heredoc_queue.empty?
+            @heredoc_queue[0][:started] = true
+            ungetc "\n"
+          end
         end
       end
-      @current_readed = @readed
+      unless tk
-      @here_readed.clear
+        case @lex_state
-      Token(TkNL)
+        when :EXPR_BEG, :EXPR_FNAME, :EXPR_DOT
+          @continue = true
+        else
+          @continue = false
+          @lex_state = :EXPR_BEG unless @escaped_nl
+          until (@indent_stack.empty? ||
+                 [TkLPAREN, TkLBRACK, TkLBRACE,
+                   TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
+            @indent_stack.pop
+          end
+        end
+        @current_readed = @readed
+        @here_readed.clear
+        tk = Token(TkNL)
+      end
+      @escaped_nl = false
+      tk
     end
 
-    @OP.def_rules("*", "**",
+    @OP.def_rules("=") do
-                  "=", "==", "===",
-                  "=~", "<=>",
-                  "<", "<=",
-                  ">", ">=", ">>") do
       |op, io|
       case @lex_state
       when :EXPR_FNAME, :EXPR_DOT
@@ -478,10 +531,39 @@ class RDoc::RubyLex
       Token(op)
     end
 
+    @OP.def_rules("*", "**",
+                  "==", "===",
+                  "=~", "<=>",
+                  "<", "<=",
+                  ">", ">=", ">>", "=>") do
+      |op, io|
+      case @lex_state
+      when :EXPR_FNAME, :EXPR_DOT
+        tk = Token(TkId, op)
+        @lex_state = :EXPR_ARG
+      else
+        tk = Token(op)
+        @lex_state = :EXPR_BEG
+      end
+      tk
+    end
+
+    @OP.def_rules("->") do
+      |op, io|
+      @lex_state = :EXPR_ENDFN
+      Token(op)
+    end
+
     @OP.def_rules("!", "!=", "!~") do
       |op, io|
-      @lex_state = :EXPR_BEG
+      case @lex_state
-      Token(op)
+      when :EXPR_FNAME, :EXPR_DOT
+        @lex_state = :EXPR_ARG
+        Token(TkId, op)
+      else
+        @lex_state = :EXPR_BEG
+        Token(op)
+      end
     end
 
     @OP.def_rules("<<") do
@@ -490,16 +572,17 @@ class RDoc::RubyLex
       if @lex_state != :EXPR_END && @lex_state != :EXPR_CLASS &&
          (@lex_state != :EXPR_ARG || @space_seen)
         c = peek(0)
-        if /\S/ =~ c && (/["'`]/ =~ c || /\w/ =~ c || c == "-")
+        if /\S/ =~ c && (/["'`]/ =~ c || /\w/ =~ c || c == "-" || c == "~")
-          tk = identify_here_document
+          tk = identify_here_document(op)
         end
       end
       unless tk
-        tk = Token(op)
         case @lex_state
         when :EXPR_FNAME, :EXPR_DOT
+          tk = Token(TkId, op)
           @lex_state = :EXPR_ARG
         else
+          tk = Token(op)
           @lex_state = :EXPR_BEG
         end
       end
@@ -513,9 +596,9 @@ class RDoc::RubyLex
 
     @OP.def_rules("`") do
       |op, io|
-      if @lex_state == :EXPR_FNAME
+      if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
-        @lex_state = :EXPR_END
+        @lex_state = :EXPR_ARG
-        Token(op)
+        Token(TkId, op)
       else
         identify_string(op)
       end
@@ -525,6 +608,7 @@ class RDoc::RubyLex
       |op, io|
       if @lex_state == :EXPR_END
         @lex_state = :EXPR_BEG
+        @after_question = true
         Token(TkQUESTION)
       else
         ch = getc
@@ -534,17 +618,31 @@ class RDoc::RubyLex
           Token(TkQUESTION)
         else
           @lex_state = :EXPR_END
+          ch << getc if "\\" == ch
           Token(TkCHAR, "?#{ch}")
         end
       end
     end
 
-    @OP.def_rules("&", "&&", "|", "||") do
+    @OP.def_rules("&&", "||") do
       |op, io|
       @lex_state = :EXPR_BEG
       Token(op)
     end
 
+    @OP.def_rules("&", "|") do
+      |op, io|
+      case @lex_state
+      when :EXPR_FNAME, :EXPR_DOT
+        tk = Token(TkId, op)
+        @lex_state = :EXPR_ARG
+      else
+        tk = Token(op)
+        @lex_state = :EXPR_BEG
+      end
+      tk
+    end
+
     @OP.def_rules("+=", "-=", "*=", "**=",
                   "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
       |op, io|
@@ -556,19 +654,22 @@ class RDoc::RubyLex
     @OP.def_rule("+@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
       |op, io|
       @lex_state = :EXPR_ARG
-      Token(op)
+      Token(TkId, op)
     end
 
     @OP.def_rule("-@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
       |op, io|
       @lex_state = :EXPR_ARG
-      Token(op)
+      Token(TkId, op)
     end
 
     @OP.def_rules("+", "-") do
       |op, io|
       catch(:RET) do
-        if @lex_state == :EXPR_ARG
+        if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
+          tk = Token(TkId, op)
+          @lex_state = :EXPR_ARG
+        elsif @lex_state == :EXPR_ARG
           if @space_seen and peek(0) =~ /[0-9]/
             throw :RET, identify_number(op)
           else
@@ -579,20 +680,21 @@ class RDoc::RubyLex
         else
           @lex_state = :EXPR_BEG
         end
-        Token(op)
+        tk = Token(op) unless tk
+        tk
       end
     end
 
-    @OP.def_rule(".") do
+    @OP.def_rules(".", "&.") do
       |op, io|
       @lex_state = :EXPR_BEG
       if peek(0) =~ /[0-9]/
         ungetc
         identify_number
       else
-        # for "obj.if" etc.
+        # for "obj.if" or "obj&.if" etc.
         @lex_state = :EXPR_DOT
-        Token(TkDOT)
+        Token(op)
       end
     end
 
@@ -639,7 +741,10 @@ class RDoc::RubyLex
 
     @OP.def_rule("/") do
       |op, io|
-      if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
+      if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
+        @lex_state = :EXPR_ARG
+        Token(TkId, op)
+      elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID || @first_in_method_statement
         identify_string(op)
       elsif peek(0) == '='
         getc
@@ -655,8 +760,15 @@ class RDoc::RubyLex
 
     @OP.def_rules("^") do
       |op, io|
-      @lex_state = :EXPR_BEG
+      case @lex_state
-      Token("^")
+      when :EXPR_FNAME, :EXPR_DOT
+        tk = Token(TkId, op)
+        @lex_state = :EXPR_ARG
+      else
+        tk = Token(op)
+        @lex_state = :EXPR_BEG
+      end
+      tk
     end
 
     #       @OP.def_rules("^=") do
@@ -683,8 +795,14 @@ class RDoc::RubyLex
 
     @OP.def_rule("~") do
       |op, io|
-      @lex_state = :EXPR_BEG
+      case @lex_state
-      Token("~")
+      when :EXPR_FNAME, :EXPR_DOT
+        @lex_state = :EXPR_ARG
+        Token(TkId, op)
+      else
+        @lex_state = :EXPR_BEG
+        Token(op)
+      end
     end
 
     @OP.def_rule("~@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
@@ -710,17 +828,18 @@ class RDoc::RubyLex
     @OP.def_rule("[]", proc{|op, io| @lex_state == :EXPR_FNAME}) do
       |op, io|
       @lex_state = :EXPR_ARG
-      Token("[]")
+      Token(TkId, op)
     end
 
     @OP.def_rule("[]=", proc{|op, io| @lex_state == :EXPR_FNAME}) do
       |op, io|
       @lex_state = :EXPR_ARG
-      Token("[]=")
+      Token(TkId, op)
     end
 
     @OP.def_rule("[") do
       |op, io|
+      text = nil
       @indent += 1
       if @lex_state == :EXPR_FNAME
         tk_c = TkfLBRACK
@@ -729,13 +848,25 @@ class RDoc::RubyLex
           tk_c = TkLBRACK
         elsif @lex_state == :EXPR_ARG && @space_seen
           tk_c = TkLBRACK
+        elsif @lex_state == :EXPR_DOT
+          if peek(0) == "]"
+            tk_c = TkIDENTIFIER
+            getc
+            if peek(0) == "="
+              text = "[]="
+            else
+              text = "[]"
+            end
+          else
+            tk_c = TkOp
+          end
         else
           tk_c = TkfLBRACK
         end
         @lex_state = :EXPR_BEG
       end
       @indent_stack.push tk_c
-      Token(tk_c)
+      Token(tk_c, text)
     end
 
     @OP.def_rule("{") do
@@ -753,23 +884,25 @@ class RDoc::RubyLex
 
     @OP.def_rule('\\') do
       |op, io|
-      if getc == "\n"
+      if peek(0) == "\n"
         @space_seen = true
         @continue = true
-        Token(TkSPACE)
+        @escaped_nl = true
-      else
-        ungetc
-        Token("\\")
       end
+      Token("\\")
     end
 
     @OP.def_rule('%') do
       |op, io|
-      if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
+      if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
+        @lex_state = :EXPR_ARG
+        Token(TkId, op)
+      elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
         identify_quotation
       elsif peek(0) == '='
         getc
-        Token(TkOPASGN, :%)
+        @lex_state = :EXPR_BEG
+        Token(TkOPASGN, '%')
       elsif @lex_state == :EXPR_ARG and @space_seen and peek(0) !~ /\s/
         identify_quotation
       else
@@ -871,7 +1004,7 @@ class RDoc::RubyLex
 
     ungetc
 
-    if (ch == "!" || ch == "?") && token[0,1] =~ /\w/ && peek(0) != "="
+    if ((ch == "!" && peek(1) != "=") || ch == "?") && token[0,1] =~ /\w/
       token.concat getc
     end
 
@@ -944,44 +1077,63 @@ class RDoc::RubyLex
             @lex_state = :EXPR_END
           end
         end
+        if token_c.ancestors.include?(TkId) and peek(0) == ':' and !peek_match?(/^::/)
+          token.concat getc
+          token_c = TkSYMBOL
+        end
         return Token(token_c, token)
       end
     end
 
     if @lex_state == :EXPR_FNAME
       @lex_state = :EXPR_END
-      if peek(0) == '='
+      if peek(0) == '=' and peek(1) != '>'
         token.concat getc
       end
     elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_DOT ||
-          @lex_state == :EXPR_ARG
+          @lex_state == :EXPR_ARG || @lex_state == :EXPR_MID
       @lex_state = :EXPR_ARG
     else
       @lex_state = :EXPR_END
     end
 
     if token[0, 1] =~ /[A-Z]/
-      return Token(TkCONSTANT, token)
+      if token[-1] =~ /[!?]/
+        token_c = TkIDENTIFIER
+      else
+        token_c = TkCONSTANT
+      end
     elsif token[token.size - 1, 1] =~ /[!?]/
-      return Token(TkFID, token)
+      token_c = TkFID
     else
-      return Token(TkIDENTIFIER, token)
+      token_c = TkIDENTIFIER
+    end
+    if peek(0) == ':' and !peek_match?(/^::/)
+      token.concat getc
+      return Token(TkSYMBOL, token)
+    else
+      return Token(token_c, token)
     end
   end
 
-  def identify_here_document
+  def identify_here_document(op)
     ch = getc
+    start_token = op
     #    if lt = PERCENT_LTYPE[ch]
-    if ch == "-"
+    if ch == "-" or ch == "~"
+      start_token.concat ch
       ch = getc
       indent = true
     end
     if /['"`]/ =~ ch
+      start_token.concat ch
       user_quote = lt = ch
       quoted = ""
       while (c = getc) && c != lt
         quoted.concat c
       end
+      start_token.concat quoted
+      start_token.concat lt
     else
       user_quote = nil
       lt = '"'
@@ -989,57 +1141,38 @@ class RDoc::RubyLex
       while (c = getc) && c =~ /\w/
         quoted.concat c
       end
+      start_token.concat quoted
       ungetc
     end
 
+    @heredoc_queue << {
+      quoted: quoted,
+      lt: lt,
+      indent: indent,
+      started: false
+    }
+    @lex_state = :EXPR_END
+    Token(RDoc::RubyLex::TkHEREDOCBEG, start_token)
+  end
+
+  def identify_here_document_body(quoted, lt, indent)
     ltback, @ltype = @ltype, lt
-    reserve = []
-    while ch = getc
-      reserve.push ch
-      if ch == "\\"
-        reserve.push ch = getc
-      elsif ch == "\n"
-        break
-      end
-    end
 
-    output_heredoc = reserve.join =~ /\A\r?\n\z/
+    doc = ""
-
+    heredoc_end = nil
-    if output_heredoc then
-      doc = '<<'
-      doc << '-' if indent
-      doc << "#{user_quote}#{quoted}#{user_quote}\n"
-    else
-      doc = '"'
-    end
-
-    @current_readed = @readed
     while l = gets
       l = l.sub(/(:?\r)?\n\z/, "\n")
       if (indent ? l.strip : l.chomp) == quoted
+        heredoc_end = l
         break
       end
       doc << l
     end
+    raise Error, "Missing terminating #{quoted} for string" unless heredoc_end
 
-    if output_heredoc then
-      raise Error, "Missing terminating #{quoted} for string" unless l
-
-      doc << l.chomp
-    else
-      doc << '"'
-    end
-
-    @current_readed = @here_readed
-    @here_readed.concat reserve
-    while ch = reserve.pop
-      ungetc ch
-    end
-
-    token_class = output_heredoc ? RDoc::RubyLex::TkHEREDOC : Ltype2Token[lt]
     @ltype = ltback
-    @lex_state = :EXPR_END
+    @lex_state = :EXPR_BEG
-    Token(token_class, doc)
+    [Token(RDoc::RubyLex::TkHEREDOC, doc), heredoc_end]
   end
 
   def identify_quotation
@@ -1066,7 +1199,7 @@ class RDoc::RubyLex
 
     num = op
 
-    if peek(0) == "0" && peek(1) !~ /[.eE]/
+    if peek(0) == "0" && peek(1) !~ /[.eEri]/
       num << getc
 
       case peek(0)
@@ -1125,6 +1258,7 @@ class RDoc::RubyLex
     type = TkINTEGER
     allow_point = true
     allow_e = true
+    allow_ri = true
     non_digit = false
     while ch = getc
       num << ch
@@ -1154,8 +1288,25 @@ class RDoc::RubyLex
           num << getc
         end
         allow_e = false
+        allow_ri = false
         allow_point = false
         non_digit = ch
+      when allow_ri && "r"
+        if non_digit
+          raise Error, "trailing `#{non_digit}' in number"
+        end
+        type = TkRATIONAL
+        if peek(0) == 'i'
+          type = TkIMAGINARY
+          num << getc
+        end
+        break
+      when allow_ri && "i"
+        if non_digit && non_digit != "r"
+          raise Error, "trailing `#{non_digit}' in number"
+        end
+        type = TkIMAGINARY
+        break
       else
         if non_digit
           raise Error, "trailing `#{non_digit}' in number"
@@ -1174,10 +1325,10 @@ class RDoc::RubyLex
     @ltype = ltype
     @quoted = quoted
 
-    str = if ltype == quoted and %w[" ' /].include? ltype then
+    str = if ltype == quoted and %w[" ' / `].include? ltype and type.nil? then
             ltype.dup
           else
-            "%#{type or PERCENT_LTYPE.key ltype}#{PERCENT_PAREN_REV[quoted]||quoted}"
+            "%#{type}#{PERCENT_PAREN_REV[quoted]||quoted}"
           end
 
     subtype = nil
@@ -1191,21 +1342,21 @@ class RDoc::RubyLex
           break
         elsif @ltype != "'" && @ltype != "]" && @ltype != ":" and ch == "#"
           ch = getc
-          subtype = true
           if ch == "{" then
+            subtype = true
             str << ch << skip_inner_expression
             next
           else
             ungetc
           end
         elsif ch == '\\'
-          if %w[' /].include? @ltype then
+          case @ltype
+          when "'" then
             case ch = getc
-            when "\\", "\n", "'"
+            when "'", '\\' then
-            when @ltype
               str << ch
             else
-              ungetc
+              str << ch
             end
           else
             str << read_escape
@@ -1227,7 +1378,10 @@ class RDoc::RubyLex
         end
       end
 
-      if subtype
+      if peek(0) == ':' and !peek_match?(/^::/) and :EXPR_BEG == @lex_state and !@after_question
+        str.concat getc
+        return Token(TkSYMBOL, str)
+      elsif subtype
         Token(DLtype2Token[ltype], str)
       else
         Token(Ltype2Token[ltype], str)

lib/rdoc/ruby_token.rb

@@ -26,7 +26,8 @@ module RDoc::RubyToken
     Symbol = Integer
   end
 
-  def set_token_position(line, char)
+  def set_token_position(seek, line, char)
+    @prev_seek = seek
     @prev_line_no = line
     @prev_char_no = char
   end
@@ -302,8 +303,8 @@ module RDoc::RubyToken
     [:TkIN,         TkKW,  "in",     :EXPR_BEG],
     [:TkDO,         TkKW,  "do",     :EXPR_BEG],
     [:TkRETURN,     TkKW,  "return", :EXPR_MID],
-    [:TkYIELD,      TkKW,  "yield",  :EXPR_END],
+    [:TkYIELD,      TkKW,  "yield",  :EXPR_ARG],
-    [:TkSUPER,      TkKW,  "super",  :EXPR_END],
+    [:TkSUPER,      TkKW,  "super",  :EXPR_ARG],
     [:TkSELF,       TkKW,  "self",   :EXPR_END],
     [:TkNIL,        TkKW,  "nil",    :EXPR_END],
     [:TkTRUE,       TkKW,  "true",   :EXPR_END],
@@ -316,11 +317,12 @@ module RDoc::RubyToken
     [:TkWHILE_MOD,  TkKW],
     [:TkUNTIL_MOD,  TkKW],
     [:TkALIAS,      TkKW,  "alias",    :EXPR_FNAME],
-    [:TkDEFINED,    TkKW,  "defined?", :EXPR_END],
+    [:TkDEFINED,    TkKW,  "defined?", :EXPR_ARG],
     [:TklBEGIN,     TkKW,  "BEGIN",    :EXPR_END],
     [:TklEND,       TkKW,  "END",      :EXPR_END],
     [:Tk__LINE__,   TkKW,  "__LINE__", :EXPR_END],
     [:Tk__FILE__,   TkKW,  "__FILE__", :EXPR_END],
+    [:Tk__ENCODING__,TkKW, "__ENCODING__", :EXPR_END],
 
     [:TkIDENTIFIER, TkId],
     [:TkFID,        TkId],
@@ -328,9 +330,13 @@ module RDoc::RubyToken
     [:TkCVAR,       TkId],
     [:TkIVAR,       TkId],
     [:TkCONSTANT,   TkId],
+    [:TkHEREDOCBEG, TkId],
+    [:TkHEREDOCEND, TkId],
 
     [:TkINTEGER,    TkVal],
     [:TkFLOAT,      TkVal],
+    [:TkRATIONAL,   TkVal],
+    [:TkIMAGINARY,  TkVal],
     [:TkSTRING,     TkVal],
     [:TkHEREDOC,    TkVal],
     [:TkXSTRING,    TkVal],
@@ -353,6 +359,7 @@ module RDoc::RubyToken
     [:TkNEQ,        TkOp,   "!="],
     [:TkGEQ,        TkOp,   ">="],
     [:TkLEQ,        TkOp,   "<="],
+    [:TkHASHROCKET, TkOp,   "=>"],
     [:TkANDOP,      TkOp,   "&&"],
     [:TkOROP,       TkOp,   "||"],
     [:TkMATCH,      TkOp,   "=~"],
@@ -367,6 +374,7 @@ module RDoc::RubyToken
     [:TkCOLON3,     TkOp,   '::'],
     #[:OPASGN,       TkOp],               # +=, -=  etc. #
     [:TkASSOC,      TkOp,   "=>"],
+    [:TkLAMBDA,     TkOp,   "->"],
     [:TkQUESTION,   TkOp,   "?"], #?
     [:TkCOLON,      TkOp,   ":"],        #:
 
@@ -394,6 +402,7 @@ module RDoc::RubyToken
 
     [:TkASSIGN,     Token,  "="],
     [:TkDOT,        Token,  "."],
+    [:TkSAFENAV,    Token,  "&."],
     [:TkLPAREN,     Token,  "("],  #(exp)
     [:TkLBRACK,     Token,  "["],  #[arry]
     [:TkLBRACE,     Token,  "{"],  #{hash}

lib/rdoc/test_case.rb

@@ -6,7 +6,7 @@ rescue NoMethodError, Gem::LoadError
 end
 
 require 'minitest/autorun'
-require 'minitest/benchmark' if ENV['BENCHMARK']
+require 'minitest/benchmark' unless ENV['NOBENCHMARK']
 
 require 'fileutils'
 require 'pp'

lib/rdoc/token_stream.rb

@@ -36,17 +36,27 @@ module RDoc::TokenStream
               when RDoc::RubyToken::TkIVAR     then 'ruby-ivar'
               when RDoc::RubyToken::TkOp       then 'ruby-operator'
               when RDoc::RubyToken::TkId       then 'ruby-identifier'
+              when RDoc::RubyToken::TkREGEXP   then 'ruby-regexp'
+              when RDoc::RubyToken::TkDREGEXP  then 'ruby-regexp'
               when RDoc::RubyToken::TkNode     then 'ruby-node'
               when RDoc::RubyToken::TkCOMMENT  then 'ruby-comment'
-              when RDoc::RubyToken::TkREGEXP   then 'ruby-regexp'
+              when RDoc::RubyToken::TkXSTRING  then 'ruby-string'
               when RDoc::RubyToken::TkSTRING   then 'ruby-string'
               when RDoc::RubyToken::TkVal      then 'ruby-value'
               end
 
-      text = CGI.escapeHTML t.text
+      comment_with_nl = false
+      case t
+      when RDoc::RubyToken::TkRD_COMMENT, RDoc::RubyToken::TkHEREDOCEND
+        comment_with_nl = true if t.text =~ /\n$/
+        text = t.text.rstrip
+      else
+        text = t.text
+      end
+      text = CGI.escapeHTML text
 
       if style then
-        "<span class=\"#{style}\">#{text}</span>"
+        "<span class=\"#{style}\">#{text}</span>#{"\n" if comment_with_nl}"
       else
         text
       end

test/rdoc/test_rdoc_code_object.rb

@@ -276,12 +276,6 @@ class TestRDocCodeObject < XrefTestCase
     assert_equal 'not_rdoc', @co.metadata['markup']
   end
 
-  def test_offset
-    @c1_m.offset = 5
-
-    assert_equal 5, @c1_m.offset
-  end
-
   def test_options
     assert_kind_of RDoc::Options, @co.options
 

test/rdoc/test_rdoc_context.rb

@@ -481,6 +481,32 @@ class TestRDocContext < XrefTestCase
     assert_equal expected_attrs, attrs
   end
 
+  def test_each_section_only_display
+    sects  = []
+    consts = []
+    attrs  = []
+
+    @c7.each_section do |section, constants, attributes|
+      sects  << section
+      consts << constants
+      attrs  << attributes
+    end
+
+    assert_equal [nil], sects.map { |section| section.title }
+
+    expected_consts = [
+      @c7.constants.select(&:display?).sort
+    ]
+
+    assert_equal expected_consts, consts
+
+    expected_attrs = [
+      @c7.attributes.select(&:display?).sort
+    ]
+
+    assert_equal expected_attrs, attrs
+  end
+
   def test_each_section_enumerator
     assert_kind_of Enumerator, @c1.each_section
   end

test/rdoc/test_rdoc_markup_to_html.rb

@@ -451,6 +451,22 @@ class TestRDocMarkupToHtml < RDoc::Markup::FormatterTestCase
     assert_equal expected, @to.res.join
   end
 
+  def test_accept_verbatim_nl_after_backslash
+    verb = @RM::Verbatim.new("a = 1 if first_flag_var and \\\n", "  this_is_flag_var\n")
+
+    @to.start_accepting
+    @to.accept_verbatim verb
+
+    expected = <<-EXPECTED
+
+<pre class="ruby"><span class="ruby-identifier">a</span> = <span class="ruby-value">1</span> <span class="ruby-keyword">if</span> <span class="ruby-identifier">first_flag_var</span> <span class="ruby-keyword">and</span> \\
+  <span class="ruby-identifier">this_is_flag_var</span>
+</pre>
+    EXPECTED
+
+    assert_equal expected, @to.res.join
+  end
+
   def test_accept_verbatim_pipe
     @options.pipe = true
 
@@ -469,6 +485,106 @@ class TestRDocMarkupToHtml < RDoc::Markup::FormatterTestCase
     assert_equal expected, @to.res.join
   end
 
+  def test_accept_verbatim_escape_in_string
+    code = <<-'RUBY'
+def foo
+  [
+    '\\',
+    '\'',
+    "'",
+    "\'\"\`",
+    "\#",
+    "\#{}",
+    "#",
+    "#{}",
+    /'"/,
+    /\'\"/,
+    /\//,
+    /\\/,
+    /\#/,
+    /\#{}/,
+    /#/,
+    /#{}/
+  ]
+end
+def bar
+end
+    RUBY
+    verb = @RM::Verbatim.new(*code.split(/(?<=\n)/))
+
+    @to.start_accepting
+    @to.accept_verbatim verb
+
+    expected = <<-'EXPECTED'
+
+<pre class="ruby"><span class="ruby-keyword">def</span> <span class="ruby-identifier">foo</span>
+  [
+    <span class="ruby-string">&#39;\\&#39;</span>,
+    <span class="ruby-string">&#39;\&#39;&#39;</span>,
+    <span class="ruby-string">&quot;&#39;&quot;</span>,
+    <span class="ruby-string">&quot;\&#39;\&quot;\`&quot;</span>,
+    <span class="ruby-string">&quot;\#&quot;</span>,
+    <span class="ruby-string">&quot;\#{}&quot;</span>,
+    <span class="ruby-string">&quot;#&quot;</span>,
+    <span class="ruby-node">&quot;#{}&quot;</span>,
+    <span class="ruby-regexp">/&#39;&quot;/</span>,
+    <span class="ruby-regexp">/\&#39;\&quot;/</span>,
+    <span class="ruby-regexp">/\//</span>,
+    <span class="ruby-regexp">/\\/</span>,
+    <span class="ruby-regexp">/\#/</span>,
+    <span class="ruby-regexp">/\#{}/</span>,
+    <span class="ruby-regexp">/#/</span>,
+    <span class="ruby-regexp">/#{}/</span>
+  ]
+<span class="ruby-keyword">end</span>
+<span class="ruby-keyword">def</span> <span class="ruby-identifier">bar</span>
+<span class="ruby-keyword">end</span>
+</pre>
+    EXPECTED
+
+    assert_equal expected, @to.res.join
+  end
+
+  def test_accept_verbatim_escape_in_backtick
+    code = <<-'RUBY'
+def foo
+  [
+    `\\`,
+    `\'\"\``,
+    `\#`,
+    `\#{}`,
+    `#`,
+    `#{}`
+  ]
+end
+def bar
+end
+    RUBY
+    verb = @RM::Verbatim.new(*code.split(/(?<=\n)/))
+
+    @to.start_accepting
+    @to.accept_verbatim verb
+
+    expected = <<-'EXPECTED'
+
+<pre class="ruby"><span class="ruby-keyword">def</span> <span class="ruby-identifier">foo</span>
+  [
+    <span class="ruby-string">`\\`</span>,
+    <span class="ruby-string">`\&#39;\&quot;\``</span>,
+    <span class="ruby-string">`\#`</span>,
+    <span class="ruby-string">`\#{}`</span>,
+    <span class="ruby-string">`#`</span>,
+    <span class="ruby-node">`#{}`</span>
+  ]
+<span class="ruby-keyword">end</span>
+<span class="ruby-keyword">def</span> <span class="ruby-identifier">bar</span>
+<span class="ruby-keyword">end</span>
+</pre>
+    EXPECTED
+
+    assert_equal expected, @to.res.join
+  end
+
   def test_accept_verbatim_ruby
     verb = @RM::Verbatim.new("1 + 1\n")
     verb.format = :ruby
@@ -485,6 +601,36 @@ class TestRDocMarkupToHtml < RDoc::Markup::FormatterTestCase
     assert_equal expected, @to.res.join
   end
 
+  def test_accept_verbatim_redefinable_operators
+    functions = %w[| ^ & <=> == === =~ > >= < <= << >> + - * / % ** ~ +@ -@ [] []= ` !  != !~].map { |redefinable_op|
+      ["def #{redefinable_op}\n", "end\n"]
+    }.flatten
+
+    verb = @RM::Verbatim.new(*functions)
+
+    @to.start_accepting
+    @to.accept_verbatim verb
+
+    expected = <<-EXPECTED
+
+<pre class="ruby">
+    EXPECTED
+    expected = expected.rstrip
+
+    %w[| ^ &amp; &lt;=&gt; == === =~ &gt; &gt;= &lt; &lt;= &lt;&lt; &gt;&gt; + - * / % ** ~ +@ -@ [] []= ` !  != !~].each do |html_escaped_op|
+      expected += <<-EXPECTED
+<span class="ruby-keyword">def</span> <span class="ruby-identifier">#{html_escaped_op}</span>
+<span class="ruby-keyword">end</span>
+      EXPECTED
+    end
+
+    expected += <<-EXPECTED
+</pre>
+EXPECTED
+
+    assert_equal expected, @to.res.join
+  end
+
   def test_convert_string
     assert_equal '&lt;&gt;', @to.convert_string('<>')
   end

test/rdoc/test_rdoc_parser_c.rb

@@ -1139,7 +1139,6 @@ Init_Foo(void) {
     assert_equal 'my_method', other_function.name
     assert_equal 'a comment for rb_other_function', other_function.comment.text
     assert_equal '()', other_function.params
-    assert_equal 118, other_function.offset
     assert_equal 8, other_function.line
 
     code = other_function.token_stream.first.text
@@ -1173,7 +1172,6 @@ Init_Foo(void) {
     assert_equal 'my_method', other_function.name
     assert_equal 'a comment for other_function', other_function.comment.text
     assert_equal '()', other_function.params
-    assert_equal 39, other_function.offset
     assert_equal 4, other_function.line
 
     code = other_function.token_stream.first.text
@@ -1402,7 +1400,6 @@ rb_m(int argc, VALUE *argv, VALUE obj) {
 
     assert_equal 'm', m.name
     assert_equal @top_level, m.file
-    assert_equal 115, m.offset
     assert_equal 7, m.line
 
     assert_equal '(p1)', m.params

test/rdoc/test_rdoc_parser_ruby.rb

@@ -74,7 +74,7 @@ class C; end
 
     comment = parser.collect_first_comment
 
-    assert_equal RDoc::Comment.new("first\n\n", @top_level), comment
+    assert_equal RDoc::Comment.new("=begin\nfirst\n=end\n\n", @top_level), comment
   end
 
   def test_get_class_or_module
@@ -282,6 +282,31 @@ class C; end
     assert_equal 'blah', @top_level.metadata['unhandled']
   end
 
+  def test_parse_for_in
+    klass = RDoc::NormalClass.new 'Foo'
+    klass.parent = @top_level
+
+    comment = RDoc::Comment.new '', @top_level
+
+    util_parser <<ruby
+def sum(n)
+  result = 0
+  for i in 1..n do
+    result += i
+  end
+  result
+end
+ruby
+
+    tk = @parser.get_tk
+
+    @parser.parse_method klass, RDoc::Parser::Ruby::NORMAL, tk, comment
+
+    sum = klass.method_list.first
+    assert_equal 'sum',      sum.name
+    assert_equal @top_level, sum.file
+  end
+
   def test_parse_alias
     klass = RDoc::NormalClass.new 'Foo'
     klass.parent = @top_level
@@ -297,7 +322,6 @@ class C; end
     assert_equal klass,      alas.parent
     assert_equal 'comment',  alas.comment
     assert_equal @top_level, alas.file
-    assert_equal 0,          alas.offset
     assert_equal 1,          alas.line
   end
 
@@ -365,7 +389,6 @@ class C; end
     assert_equal 'foo', foo.name
     assert_equal 'my attr', foo.comment.text
     assert_equal @top_level, foo.file
-    assert_equal 0, foo.offset
     assert_equal 1, foo.line
   end
 
@@ -404,7 +427,6 @@ class C; end
     assert_equal 'RW', foo.rw
     assert_equal 'my attr', foo.comment.text
     assert_equal @top_level, foo.file
-    assert_equal 0, foo.offset
     assert_equal 1, foo.line
 
     bar = klass.attributes.last
@@ -617,7 +639,6 @@ class C; end
     assert_equal 'Foo', foo.full_name
     assert_equal 'my class', foo.comment.text
     assert_equal [@top_level], foo.in_files
-    assert_equal 0, foo.offset
     assert_equal 1, foo.line
   end
 
@@ -637,7 +658,6 @@ end
 
     c = @top_level.classes.first
     assert_equal 'C', c.full_name
-    assert_equal 0, c.offset
     assert_equal 1, c.line
   end
 
@@ -727,7 +747,6 @@ end
     assert_equal 'Foo', foo.full_name
     assert_empty foo.comment
     assert_equal [@top_level], foo.in_files
-    assert_equal 0, foo.offset
     assert_equal 1, foo.line
   end
 
@@ -758,6 +777,16 @@ end
     assert_empty @top_level.classes.first.comment
   end
 
+  def test_parse_class_lower_name_warning
+    @options.verbosity = 2
+    out, err = capture_io do
+      util_parser "class foo\nend"
+      tk = @parser.get_tk
+      @parser.parse_class @top_level, RDoc::Parser::Ruby::NORMAL, tk, @comment
+    end
+    assert_match /Expected class name or '<<'\. Got/, err
+  end
+
   def test_parse_multi_ghost_methods
     util_parser <<-'CLASS'
 class Foo
@@ -920,7 +949,6 @@ end
     assert_equal %w[A::B A::d], modules.map { |c| c.full_name }
 
     b = modules.first
-    assert_equal 10, b.offset
     assert_equal 2,  b.line
 
     # make sure method/alias was not added to enclosing class/module
@@ -1083,7 +1111,6 @@ EOF
     assert_equal 'RW',       foo.rw
     assert_equal 'my attr',  foo.comment.text
     assert_equal @top_level, foo.file
-    assert_equal 0,          foo.offset
     assert_equal 1,          foo.line
 
     assert_equal nil,        foo.viewer
@@ -1147,7 +1174,6 @@ EOF
     assert_equal 'foo',       foo.name
     assert_equal 'my method', foo.comment.text
     assert_equal @top_level,  foo.file
-    assert_equal 0,           foo.offset
     assert_equal 1,           foo.line
 
     assert_equal [],        foo.aliases
@@ -1223,7 +1249,6 @@ EOF
 
     assert_equal 'A', foo.name
     assert_equal @top_level, foo.file
-    assert_equal 0, foo.offset
     assert_equal 1, foo.line
   end
 
@@ -1389,7 +1414,6 @@ A::B::C = 1
     assert_equal 'foo',       foo.name
     assert_equal 'my method', foo.comment.text
     assert_equal @top_level,  foo.file
-    assert_equal 0,           foo.offset
     assert_equal 1,           foo.line
 
     assert_equal [],      foo.aliases
@@ -1587,7 +1611,6 @@ end
     assert_equal 'foo',       foo.name
     assert_equal 'my method', foo.comment.text
     assert_equal @top_level,  foo.file
-    assert_equal 0,           foo.offset
     assert_equal 1,           foo.line
 
     assert_equal [],        foo.aliases
@@ -1627,6 +1650,47 @@ end
     assert_equal stream, foo.token_stream
   end
 
+  def test_parse_redefinable_methods
+    klass = RDoc::NormalClass.new 'Foo'
+    klass.parent = @top_level
+
+    comment = RDoc::Comment.new "", @top_level
+
+    redefinable_ops = %w[| ^ & <=> == === =~ > >= < <= << >> + - * / % ** ~ +@ -@ [] []= ` !  != !~]
+    redefinable_ops.each do |redefinable_op|
+      util_parser "def #{redefinable_op}\nend\n"
+      tk = @parser.get_tk
+      @parser.parse_method klass, RDoc::Parser::Ruby::NORMAL, tk, comment
+    end
+
+    klass.method_list.each do |method|
+      assert_kind_of RDoc::RubyToken::TkId, method.token_stream[5]
+      assert_includes redefinable_ops, method.token_stream[5].text
+    end
+  end
+
+  def test_parse_method_bracket
+    util_parser <<-RUBY
+class C
+  def [] end
+  def self.[] end
+  def []= end
+  def self.[]= end
+end
+    RUBY
+
+    @parser.scan
+
+    c = @store.find_class_named 'C'
+
+    assert_equal 4, c.method_list.size
+    assert_equal 'C#[]', c.method_list[0].full_name
+    assert_equal 'C::[]', c.method_list[1].full_name
+    assert_equal 'C#[]=', c.method_list[2].full_name
+    assert_equal 'C::[]=', c.method_list[3].full_name
+    assert c.aliases.empty?
+  end
+
   def test_parse_method_alias
     klass = RDoc::NormalClass.new 'Foo'
     klass.parent = @top_level
@@ -1978,6 +2042,30 @@ end
     assert_equal 'Foo#blah', methods.first.full_name
   end
 
+  def test_parse_statements_postfix_if_unless
+    util_parser <<-CODE
+class C
+  def foo
+    1 if nil
+  end
+
+  def bar
+    2 unless nil
+  end
+end
+    CODE
+
+    @parser.parse_statements @top_level, RDoc::Parser::Ruby::NORMAL, nil
+
+    c = @top_level.classes.first
+    assert_equal 'C', c.full_name, 'class C'
+
+    methods = c.method_list
+    assert_equal 2, methods.length
+    assert_equal 'C#foo', methods[0].full_name
+    assert_equal 'C#bar', methods[1].full_name
+  end
+
   def test_parse_statements_class_nested
     comment = RDoc::Comment.new "##\n# my method\n", @top_level
 
@@ -1994,7 +2082,7 @@ end
   end
 
   def test_parse_statements_def_percent_string_pound
-    util_parser "class C\ndef a\n%r{#}\nend\ndef b() end\nend"
+    util_parser "class C\ndef a\n%r{#}\n%r{\#{}}\nend\ndef b() end\nend"
 
     @parser.parse_statements @top_level, RDoc::Parser::Ruby::NORMAL
 
@@ -2011,9 +2099,11 @@ end
       tk(:SPACE,      11, 2, 3, nil,   ' '),
       tk(:IDENTIFIER, 12, 2, 4, 'a',   'a'),
       tk(:NL,         13, 2, 5, nil,   "\n"),
-      tk(:DREGEXP,    14, 3, 0, nil,   '%r{#}'),
+      tk(:REGEXP,     14, 3, 0, nil,   '%r{#}'),
       tk(:NL,         19, 3, 5, nil,   "\n"),
-      tk(:END,        20, 4, 0, 'end', 'end'),
+      tk(:DREGEXP,    20, 4, 0, nil,   '%r{#{}}'),
+      tk(:NL,         27, 4, 7, nil,   "\n"),
+      tk(:END,        28, 5, 0, 'end', 'end'),
     ]
 
     assert_equal expected, a.token_stream
@@ -2377,6 +2467,162 @@ end
     assert_equal :private, date_time_now.visibility, date_time_now.full_name
   end
 
+  def test_parse_statements_complex_condition_in_for
+    util_parser <<RUBY
+class Foo
+  def blah()
+    for i in (k)...n do
+    end
+    for i in (k)...n
+    end
+  end
+end
+RUBY
+
+    expected = <<EXPTECTED
+<span class="ruby-keyword">def</span> <span class="ruby-identifier">blah</span>()
+  <span class="ruby-keyword">for</span> <span class="ruby-identifier">i</span> <span class="ruby-keyword">in</span> (<span class="ruby-identifier">k</span>)<span class="ruby-operator">...</span><span class="ruby-identifier">n</span> <span class="ruby-keyword">do</span>
+  <span class="ruby-keyword">end</span>
+  <span class="ruby-keyword">for</span> <span class="ruby-identifier">i</span> <span class="ruby-keyword">in</span> (<span class="ruby-identifier">k</span>)<span class="ruby-operator">...</span><span class="ruby-identifier">n</span>
+  <span class="ruby-keyword">end</span>
+<span class="ruby-keyword">end</span>
+EXPTECTED
+    expected = expected.rstrip
+
+    @parser.scan
+
+    foo = @top_level.classes.first
+    assert_equal 'Foo', foo.full_name
+
+    blah = foo.method_list.first
+    markup_code = blah.markup_code.sub(/^.*\n/, '')
+    assert_equal markup_code, expected
+  end
+
+  def test_parse_statements_postfix_if_after_heredocbeg
+    @filename = 'file.rb'
+    util_parser <<RUBY
+class Foo
+  def blah()
+    <<~EOM if true
+    EOM
+  end
+end
+RUBY
+
+    expected = <<EXPTECTED
+  <span class="ruby-keyword">def</span> <span class="ruby-identifier">blah</span>()
+    <span class="ruby-identifier">&lt;&lt;~EOM</span> <span class="ruby-keyword">if</span> <span class="ruby-keyword">true</span>
+<span class="ruby-value"></span><span class="ruby-identifier">    EOM</span>
+  <span class="ruby-keyword">end</span>
+EXPTECTED
+    expected = expected.rstrip
+
+    @parser.scan
+
+    foo = @top_level.classes.first
+    assert_equal 'Foo', foo.full_name
+
+    blah = foo.method_list.first
+    markup_code = blah.markup_code.sub(/^.*\n/, '')
+    assert_equal markup_code, expected
+  end
+
+  def test_parse_statements_method_oneliner_with_regexp
+    util_parser <<RUBY
+class Foo
+  def blah() /bar/ end
+end
+RUBY
+
+    expected = <<EXPTECTED
+<span class="ruby-keyword">def</span> <span class="ruby-identifier">blah</span>() <span class="ruby-regexp">/bar/</span> <span class="ruby-keyword">end</span>
+EXPTECTED
+    expected = expected.rstrip
+
+    @parser.scan
+
+    foo = @top_level.classes.first
+    assert_equal 'Foo', foo.full_name
+
+    blah = foo.method_list.first
+    markup_code = blah.markup_code.sub(/^.*\n/, '')
+    assert_equal expected, markup_code
+  end
+
+  def test_parse_statements_embdoc_in_document
+    @filename = 'file.rb'
+    util_parser <<RUBY
+class Foo
+  # doc
+  #
+  #   =begin
+  #   test embdoc
+  #   =end
+  #
+  def blah
+  end
+end
+RUBY
+
+    expected = <<EXPTECTED
+<p>doc
+
+<pre class="ruby"><span class="ruby-comment">=begin
+test embdoc
+=end</span>
+</pre>
+EXPTECTED
+
+    @parser.scan
+
+    foo = @top_level.classes.first
+    assert_equal 'Foo', foo.full_name
+
+    blah = foo.method_list.first
+    markup_comment = blah.search_record[6]
+    assert_equal markup_comment, expected
+  end
+
+  def test_parse_require_dynamic_string
+    content = <<-RUBY
+prefix = 'path'
+require "\#{prefix}/a_library"
+require 'test'
+RUBY
+
+    util_parser content
+
+    @parser.parse_statements @top_level
+
+    assert_equal 1, @top_level.requires.length
+  end
+
+  def test_parse_postfix_nodoc
+    util_parser <<-RUBY
+class A
+end # :nodoc:
+
+class B
+  def a
+  end # :nodoc:
+
+  def b
+  end
+end
+RUBY
+
+    @parser.parse_statements @top_level
+
+    c_a = @top_level.classes.select(&:document_self).first
+    assert_equal 'B', c_a.full_name
+
+    assert_equal 2, @top_level.classes.length
+    assert_equal 1, @top_level.classes.count(&:document_self)
+    assert_equal 1, c_a.method_list.length
+    assert_equal 'B#b', c_a.method_list.first.full_name
+  end
+
   def test_parse_statements_identifier_require
     content = "require 'bar'"
 
@@ -2504,6 +2750,25 @@ end
     assert_equal 'A#b', m_b.full_name
   end
 
+
+  def test_parse_symbol_in_paren_arg
+    util_parser <<RUBY
+class Foo
+  def blah
+  end
+  private(:blah)
+end
+RUBY
+
+    @parser.scan
+
+    foo = @top_level.classes.first
+    assert_equal 'Foo', foo.full_name
+
+    blah = foo.method_list.first
+    assert_equal :private, blah.visibility
+  end
+
   def test_parse_symbol_in_arg
     util_parser ':blah "blah" "#{blah}" blah'
 
@@ -2736,7 +3001,7 @@ end
   def test_sanity_interpolation_curly
     util_parser '%{ #{} }'
 
-    assert_equal '%Q{ #{} }', @parser.get_tk.text
+    assert_equal '%{ #{} }', @parser.get_tk.text
     assert_equal RDoc::RubyToken::TkNL, @parser.get_tk.class
   end
 
@@ -2793,11 +3058,11 @@ end
 
     foo = @top_level.classes.first
 
-    assert_equal 'Foo comment', foo.comment.text
+    assert_equal "=begin rdoc\nFoo comment\n=end", foo.comment.text
 
     m = foo.method_list.first
 
-    assert_equal 'm comment', m.comment.text
+    assert_equal "=begin\nm comment\n=end", m.comment.text
   end
 
   def test_scan_block_comment_nested # Issue #41
@@ -2819,7 +3084,7 @@ end
     foo = @top_level.modules.first
 
     assert_equal 'Foo', foo.full_name
-    assert_equal 'findmeindoc', foo.comment.text
+    assert_equal "=begin rdoc\nfindmeindoc\n=end", foo.comment.text
 
     bar = foo.classes.first
 
@@ -2866,12 +3131,12 @@ end
 
     foo = @top_level.classes.first
 
-    assert_equal "= DESCRIPTION\n\nThis is a simple test class\n\n= RUMPUS\n\nIs a silly word",
+    assert_equal "=begin rdoc\n\n= DESCRIPTION\n\nThis is a simple test class\n\n= RUMPUS\n\nIs a silly word\n\n=end",
       foo.comment.text
 
     m = foo.method_list.first
 
-    assert_equal 'A nice girl', m.comment.text
+    assert_equal "=begin rdoc\nA nice girl\n=end", m.comment.text
   end
 
   def test_scan_class_nested_nodoc
@@ -3245,6 +3510,33 @@ end
     assert c_b.singleton
   end
 
+  def test_scan_visibility_count
+    util_parser <<-RUBY
+class C < Original::Base
+  class C2 < Original::Base
+    def m0() end
+    def m1() end
+
+    private
+
+    def m2() end
+    def m3() end
+    def m4() end
+  end
+end
+    RUBY
+
+    @parser.scan
+
+    c = @store.find_class_named 'C::C2'
+
+    private_method_count = c.method_list.count { |m| :private == m.visibility }
+    assert_equal 3, private_method_count
+
+    public_method_count = c.method_list.count { |m| :public == m.visibility }
+    assert_equal 2, public_method_count
+  end
+
   def test_singleton_method_via_eigenclass
     util_parser <<-RUBY
 class C

test/rdoc/test_rdoc_ruby_lex.rb

@@ -9,6 +9,31 @@ class TestRDocRubyLex < RDoc::TestCase
     @TK = RDoc::RubyToken
   end
 
+  def test_token_position
+    tokens = RDoc::RubyLex.tokenize '[ 1, :a, nil ]', nil
+
+    assert_equal '[', tokens[0].text
+    assert_equal 0, tokens[0].seek
+    assert_equal 1, tokens[0].line_no
+    assert_equal 0, tokens[0].char_no
+    assert_equal '1', tokens[2].text
+    assert_equal 2, tokens[2].seek
+    assert_equal 1, tokens[2].line_no
+    assert_equal 2, tokens[2].char_no
+    assert_equal ':a', tokens[5].text
+    assert_equal 5, tokens[5].seek
+    assert_equal 1, tokens[5].line_no
+    assert_equal 5, tokens[5].char_no
+    assert_equal 'nil', tokens[8].text
+    assert_equal 9, tokens[8].seek
+    assert_equal 1, tokens[8].line_no
+    assert_equal 9, tokens[8].char_no
+    assert_equal ']', tokens[10].text
+    assert_equal 13, tokens[10].seek
+    assert_equal 1, tokens[10].line_no
+    assert_equal 13, tokens[10].char_no
+  end
+
   def test_class_tokenize
     tokens = RDoc::RubyLex.tokenize "def x() end", nil
 
@@ -37,17 +62,39 @@ class TestRDocRubyLex < RDoc::TestCase
     assert_equal expected, tokens
   end
 
-  def test_class_tokenize_character_literal
+  def test_class_tokenize___ENCODING__
-    tokens = RDoc::RubyLex.tokenize "?\\", nil
+    tokens = RDoc::RubyLex.tokenize '__ENCODING__', nil
 
     expected = [
-      @TK::TkCHAR.new( 0, 1,  0, "?\\"),
+      @TK::Tk__ENCODING__.new( 0, 1,  0, '__ENCODING__'),
+      @TK::TkNL          .new(12, 1, 12, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_character_literal
+    tokens = RDoc::RubyLex.tokenize "?c", nil
+
+    expected = [
+      @TK::TkCHAR.new( 0, 1,  0, "?c"),
       @TK::TkNL  .new( 2, 1,  2, "\n"),
     ]
 
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_character_literal_with_escape
+    tokens = RDoc::RubyLex.tokenize "?\\s", nil
+
+    expected = [
+      @TK::TkCHAR.new( 0, 1,  0, "?\\s"),
+      @TK::TkNL  .new( 3, 1,  3, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_def_heredoc
     tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
 def x
@@ -64,9 +111,11 @@ end
       @TK::TkIDENTIFIER.new( 4, 1,  4, 'x'),
       @TK::TkNL        .new( 5, 1,  5, "\n"),
       @TK::TkSPACE     .new( 6, 2,  0, '  '),
-      @TK::TkHEREDOC   .new( 8, 2,  2,
+
-                            %Q{<<E\nLine 1\nLine 2\nE}),
+      @TK::TkHEREDOCBEG.new( 8, 2,  2, '<<E'),
-      @TK::TkNL        .new(27, 5, 28, "\n"),
+      @TK::TkNL        .new(11, 2,  6, "\n"),
+      @TK::TkHEREDOC   .new(11, 2,  6, "Line 1\nLine 2\n"),
+      @TK::TkHEREDOCEND.new(27, 5, 26, "E\n"),
       @TK::TkEND       .new(28, 6,  0, 'end'),
       @TK::TkNL        .new(31, 6, 28, "\n"),
     ]
@@ -74,18 +123,89 @@ end
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_reserved_keyword_with_args
+    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
+yield :foo
+super :bar
+defined? :baz
+    RUBY
+
+    expected = [
+      @TK::TkYIELD  .new( 0, 1,  0, "yield"),
+      @TK::TkSPACE  .new( 5, 1,  5, " "),
+      @TK::TkSYMBOL .new( 6, 1,  6, ":foo"),
+      @TK::TkNL     .new(10, 1, 10,  "\n"),
+      @TK::TkSUPER  .new(11, 2,  0, "super"),
+      @TK::TkSPACE  .new(16, 2,  5, " "),
+      @TK::TkSYMBOL .new(17, 2,  6, ":bar"),
+      @TK::TkNL     .new(21, 2, 11,  "\n"),
+      @TK::TkDEFINED.new(22, 3,  0, "defined?"),
+      @TK::TkSPACE  .new(30, 3,  8, " "),
+      @TK::TkSYMBOL .new(31, 3,  9, ":baz"),
+      @TK::TkNL     .new(35, 3, 22,  "\n")
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_hash_symbol
     tokens = RDoc::RubyLex.tokenize '{ class:"foo" }', nil
 
+    expected = [
+      @TK::TkLBRACE.new( 0, 1,  0, '{'),
+      @TK::TkSPACE .new( 1, 1,  1, ' '),
+      @TK::TkSYMBOL.new( 2, 1,  2, 'class:'),
+      @TK::TkSTRING.new( 8, 1,  8, '"foo"'),
+      @TK::TkSPACE .new(13, 1, 13, ' '),
+      @TK::TkRBRACE.new(14, 1, 14, '}'),
+      @TK::TkNL    .new(15, 1, 15, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_double_colon_is_not_hash_symbol
+    tokens = RDoc::RubyLex.tokenize 'self.class::Row', nil
+
+    expected = [
+      @TK::TkSELF      .new( 0, 1,  0, "self"),
+      @TK::TkDOT       .new( 4, 1,  4, "."),
+      @TK::TkIDENTIFIER.new( 5, 1,  5, "class"),
+      @TK::TkCOLON2    .new(10, 1, 10, "::"),
+      @TK::TkCONSTANT  .new(12, 1, 12, "Row"),
+      @TK::TkNL        .new(15, 1, 15, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_safe_nav_operator
+    tokens = RDoc::RubyLex.tokenize 'receiver&.meth', nil
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1,  0, "receiver"),
+      @TK::TkSAFENAV   .new( 8, 1,  8, "&."),
+      @TK::TkIDENTIFIER.new(10, 1, 10, "meth"),
+      @TK::TkNL        .new(14, 1, 14, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_hash_rocket
+    tokens = RDoc::RubyLex.tokenize '{ :class => "foo" }', nil
+
     expected = [
       @TK::TkLBRACE    .new( 0, 1,  0, '{'),
       @TK::TkSPACE     .new( 1, 1,  1, ' '),
-      @TK::TkIDENTIFIER.new( 2, 1,  2, 'class'),
+      @TK::TkSYMBOL    .new( 2, 1,  2, ':class'),
-      @TK::TkSYMBEG    .new( 7, 1,  7, ':'),
+      @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkSTRING    .new( 8, 1,  8, '"foo"'),
+      @TK::TkHASHROCKET.new( 9, 1,  9, '=>'),
-      @TK::TkSPACE     .new(13, 1, 13, ' '),
+      @TK::TkSPACE     .new(11, 1, 11, ' '),
-      @TK::TkRBRACE    .new(14, 1, 14, '}'),
+      @TK::TkSTRING    .new(12, 1, 12, '"foo"'),
-      @TK::TkNL        .new(15, 1, 15, "\n"),
+      @TK::TkSPACE     .new(17, 1, 17, ' '),
+      @TK::TkRBRACE    .new(18, 1, 18, '}'),
+      @TK::TkNL        .new(19, 1, 19, "\n"),
     ]
 
     assert_equal expected, tokens
@@ -104,10 +224,106 @@ Line 2\r
       @TK::TkSPACE     .new( 6, 1,  6, ' '),
       @TK::TkASSIGN    .new( 7, 1,  7, '='),
       @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkHEREDOC   .new( 9, 1,  9,
+      @TK::TkHEREDOCBEG.new( 9, 1,  9, '<<-STRING'),
-                            %Q{<<-STRING\nLine 1\nLine 2\n  STRING}),
+      @TK::TkSPACE     .new(18, 1, 18, "\r"),
-      @TK::TkSPACE     .new(44, 4, 45, "\r"),
+      @TK::TkNL        .new(19, 1, 19, "\n"),
-      @TK::TkNL        .new(45, 4, 46, "\n"),
+      @TK::TkHEREDOC   .new(19, 1, 19,
+                            %Q{Line 1\nLine 2\n}),
+      @TK::TkHEREDOCEND.new(45, 4, 36, "  STRING\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_opassign
+    tokens = RDoc::RubyLex.tokenize <<'RUBY', nil
+a %= b
+a /= b
+a -= b
+a += b
+a |= b
+a &= b
+a >>= b
+a <<= b
+a *= b
+a &&= b
+a ||= b
+a **= b
+RUBY
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1, 0, "a"),
+      @TK::TkSPACE     .new( 1, 1, 1, " "),
+      @TK::TkOPASGN    .new( 2, 1, 2, "%"),
+      @TK::TkSPACE     .new( 4, 1, 4, " "),
+      @TK::TkIDENTIFIER.new( 5, 1, 5, "b"),
+      @TK::TkNL        .new( 6, 1, 6, "\n"),
+      @TK::TkIDENTIFIER.new( 7, 2, 0, "a"),
+      @TK::TkSPACE     .new( 8, 2, 1, " "),
+      @TK::TkOPASGN    .new( 9, 2, 2, "/"),
+      @TK::TkSPACE     .new( 11, 2, 4, " "),
+      @TK::TkIDENTIFIER.new( 12, 2, 5, "b"),
+      @TK::TkNL        .new( 13, 2, 7, "\n"),
+      @TK::TkIDENTIFIER.new( 14, 3, 0, "a"),
+      @TK::TkSPACE     .new( 15, 3, 1, " "),
+      @TK::TkOPASGN    .new( 16, 3, 2, "-"),
+      @TK::TkSPACE     .new( 18, 3, 4, " "),
+      @TK::TkIDENTIFIER.new( 19, 3, 5, "b"),
+      @TK::TkNL        .new( 20, 3, 14, "\n"),
+      @TK::TkIDENTIFIER.new( 21, 4, 0, "a"),
+      @TK::TkSPACE     .new( 22, 4, 1, " "),
+      @TK::TkOPASGN    .new( 23, 4, 2, "+"),
+      @TK::TkSPACE     .new( 25, 4, 4, " "),
+      @TK::TkIDENTIFIER.new( 26, 4, 5, "b"),
+      @TK::TkNL        .new( 27, 4, 21, "\n"),
+      @TK::TkIDENTIFIER.new( 28, 5, 0, "a"),
+      @TK::TkSPACE     .new( 29, 5, 1, " "),
+      @TK::TkOPASGN    .new( 30, 5, 2, "|"),
+      @TK::TkSPACE     .new( 32, 5, 4, " "),
+      @TK::TkIDENTIFIER.new( 33, 5, 5, "b"),
+      @TK::TkNL        .new( 34, 5, 28, "\n"),
+      @TK::TkIDENTIFIER.new( 35, 6, 0, "a"),
+      @TK::TkSPACE     .new( 36, 6, 1, " "),
+      @TK::TkOPASGN    .new( 37, 6, 2, "&"),
+      @TK::TkSPACE     .new( 39, 6, 4, " "),
+      @TK::TkIDENTIFIER.new( 40, 6, 5, "b"),
+      @TK::TkNL        .new( 41, 6, 35, "\n"),
+      @TK::TkIDENTIFIER.new( 42, 7, 0, "a"),
+      @TK::TkSPACE     .new( 43, 7, 1, " "),
+      @TK::TkOPASGN    .new( 44, 7, 2, ">>"),
+      @TK::TkSPACE     .new( 47, 7, 5, " "),
+      @TK::TkIDENTIFIER.new( 48, 7, 6, "b"),
+      @TK::TkNL        .new( 49, 7, 42, "\n"),
+      @TK::TkIDENTIFIER.new( 50, 8, 0, "a"),
+      @TK::TkSPACE     .new( 51, 8, 1, " "),
+      @TK::TkOPASGN    .new( 52, 8, 2, "<<"),
+      @TK::TkSPACE     .new( 55, 8, 5, " "),
+      @TK::TkIDENTIFIER.new( 56, 8, 6, "b"),
+      @TK::TkNL        .new( 57, 8, 50, "\n"),
+      @TK::TkIDENTIFIER.new( 58, 9, 0, "a"),
+      @TK::TkSPACE     .new( 59, 9, 1, " "),
+      @TK::TkOPASGN    .new( 60, 9, 2, "*"),
+      @TK::TkSPACE     .new( 62, 9, 4, " "),
+      @TK::TkIDENTIFIER.new( 63, 9, 5, "b"),
+      @TK::TkNL        .new( 64, 9, 58, "\n"),
+      @TK::TkIDENTIFIER.new( 65, 10, 0, "a"),
+      @TK::TkSPACE     .new( 66, 10, 1, " "),
+      @TK::TkOPASGN    .new( 67, 10, 2, "&&"),
+      @TK::TkSPACE     .new( 70, 10, 5, " "),
+      @TK::TkIDENTIFIER.new( 71, 10, 6, "b"),
+      @TK::TkNL        .new( 72, 10, 65, "\n"),
+      @TK::TkIDENTIFIER.new( 73, 11, 0, "a"),
+      @TK::TkSPACE     .new( 74, 11, 1, " "),
+      @TK::TkOPASGN    .new( 75, 11, 2, "||"),
+      @TK::TkSPACE     .new( 78, 11, 5, " "),
+      @TK::TkIDENTIFIER.new( 79, 11, 6, "b"),
+      @TK::TkNL        .new( 80, 11, 73, "\n"),
+      @TK::TkIDENTIFIER.new( 81, 12, 0, "a"),
+      @TK::TkSPACE     .new( 82, 12, 1, " "),
+      @TK::TkOPASGN    .new( 83, 12, 2, "**"),
+      @TK::TkSPACE     .new( 86, 12, 5, " "),
+      @TK::TkIDENTIFIER.new( 87, 12, 6, "b"),
+      @TK::TkNL        .new( 88, 12, 81, "\n"),
     ]
 
     assert_equal expected, tokens
@@ -126,10 +342,12 @@ Line 2
       @TK::TkSPACE     .new( 6, 1,  6, ' '),
       @TK::TkASSIGN    .new( 7, 1,  7, '='),
       @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkSTRING    .new( 9, 1,  9, %Q{"Line 1\nLine 2\n"}),
+      @TK::TkHEREDOCBEG.new( 9, 1,  9, '<<-STRING'),
-      @TK::TkDOT       .new(41, 4, 42, '.'),
+      @TK::TkDOT       .new(18, 1, 18, '.'),
-      @TK::TkIDENTIFIER.new(42, 4, 43, 'chomp'),
+      @TK::TkIDENTIFIER.new(19, 1, 19, 'chomp'),
-      @TK::TkNL        .new(47, 4, 48, "\n"),
+      @TK::TkNL        .new(24, 1, 24, "\n"),
+      @TK::TkHEREDOC   .new(24, 1, 24, "Line 1\nLine 2\n"),
+      @TK::TkHEREDOCEND.new(47, 4, 39, "  STRING\n"),
     ]
 
     assert_equal expected, tokens
@@ -148,9 +366,12 @@ Line 2
       @TK::TkSPACE     .new( 6, 1,  6, ' '),
       @TK::TkASSIGN    .new( 7, 1,  7, '='),
       @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkHEREDOC   .new( 9, 1,  9,
+
-                            %Q{<<-STRING\nLine 1\nLine 2\n  STRING}),
+
-      @TK::TkNL        .new(41, 4, 42, "\n"),
+      @TK::TkHEREDOCBEG.new( 9, 1,  9, '<<-STRING'),
+      @TK::TkNL        .new(18, 1, 18, "\n"),
+      @TK::TkHEREDOC   .new(18, 1, 18, "Line 1\nLine 2\n"),
+      @TK::TkHEREDOCEND.new(41, 4, 33, "  STRING\n")
     ]
 
     assert_equal expected, tokens
@@ -180,8 +401,10 @@ U
       @TK::TkSPACE     .new( 1, 1,  1, ' '),
       @TK::TkIDENTIFIER.new( 2, 1,  2, 'b'),
       @TK::TkSPACE     .new( 3, 1,  3, ' '),
-      @TK::TkHEREDOC   .new( 4, 1,  4, %Q{<<-U\n%N\nU}),
+      @TK::TkHEREDOCBEG.new( 4, 1,  4, '<<-U'),
-      @TK::TkNL        .new(13, 3, 14, "\n"),
+      @TK::TkNL        .new( 8, 1,  8, "\n"),
+      @TK::TkHEREDOC   .new( 8, 1,  8, "%N\n"),
+      @TK::TkHEREDOCEND.new(13, 3, 12, "U\n")
     ]
 
     assert_equal expected, tokens
@@ -195,6 +418,36 @@ U
     assert_equal expected, tokens.first
   end
 
+  def test_class_tokenize_lambda
+    tokens = RDoc::RubyLex.tokenize 'a = -> x, y { x + y }', nil
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1,  0, 'a'),
+      @TK::TkSPACE     .new( 1, 1,  1, ' '),
+      @TK::TkASSIGN    .new( 2, 1,  2, '='),
+      @TK::TkSPACE     .new( 3, 1,  3, ' '),
+      @TK::TkLAMBDA    .new( 4, 1,  4, '->'),
+      @TK::TkSPACE     .new( 6, 1,  6, ' '),
+      @TK::TkIDENTIFIER.new( 7, 1,  7, 'x'),
+      @TK::TkCOMMA     .new( 8, 1,  8, ','),
+      @TK::TkSPACE     .new( 9, 1,  9, ' '),
+      @TK::TkIDENTIFIER.new(10, 1, 10, 'y'),
+      @TK::TkSPACE     .new(11, 1, 11, ' '),
+      @TK::TkfLBRACE   .new(12, 1, 12, '{'),
+      @TK::TkSPACE     .new(13, 1, 13, ' '),
+      @TK::TkIDENTIFIER.new(14, 1, 14, 'x'),
+      @TK::TkSPACE     .new(15, 1, 15, ' '),
+      @TK::TkPLUS      .new(16, 1, 16, '+'),
+      @TK::TkSPACE     .new(17, 1, 17, ' '),
+      @TK::TkIDENTIFIER.new(18, 1, 18, 'y'),
+      @TK::TkSPACE     .new(19, 1, 19, ' '),
+      @TK::TkRBRACE    .new(20, 1, 20, '}'),
+      @TK::TkNL        .new(21, 1, 21, "\n")
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_percent_1
     tokens = RDoc::RubyLex.tokenize 'v%10==10', nil
 
@@ -221,6 +474,39 @@ U
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_percent_r_with_slash
+    tokens = RDoc::RubyLex.tokenize '%r/hi/', nil
+
+    expected = [
+      @TK::TkREGEXP.new( 0, 1,  0, '%r/hi/'),
+      @TK::TkNL    .new( 6, 1, 6, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_percent_large_q
+    tokens = RDoc::RubyLex.tokenize '%Q/hi/', nil
+
+    expected = [
+      @TK::TkSTRING.new( 0, 1,  0, '%Q/hi/'),
+      @TK::TkNL    .new( 6, 1, 6, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_percent_large_q_with_double_quote
+    tokens = RDoc::RubyLex.tokenize '%Q"hi"', nil
+
+    expected = [
+      @TK::TkSTRING.new( 0, 1,  0, '%Q"hi"'),
+      @TK::TkNL    .new( 6, 1, 6, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_percent_w
     tokens = RDoc::RubyLex.tokenize '%w[hi]', nil
 
@@ -243,6 +529,35 @@ U
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_hash_rocket
+    tokens = RDoc::RubyLex.tokenize "{ :foo=> 1 }", nil
+
+    expected = [
+      @TK::TkLBRACE    .new( 0, 1,  0, '{'),
+      @TK::TkSPACE     .new( 1, 1,  1, ' '),
+      @TK::TkSYMBOL    .new( 2, 1,  2, ':foo'),
+      @TK::TkHASHROCKET.new( 6, 1,  6, '=>'),
+      @TK::TkSPACE     .new( 8, 1,  8, ' '),
+      @TK::TkINTEGER   .new( 9, 1,  9, '1'),
+      @TK::TkSPACE     .new(10, 1, 10, ' '),
+      @TK::TkRBRACE    .new(11, 1, 11, '}'),
+      @TK::TkNL        .new(12, 1, 12, "\n")
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_percent_sign_quote
+    tokens = RDoc::RubyLex.tokenize '%%hi%', nil
+
+    expected = [
+      @TK::TkSTRING.new( 0, 1, 0, '%%hi%'),
+      @TK::TkNL    .new( 5, 1, 5, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_regexp
     tokens = RDoc::RubyLex.tokenize "/hay/", nil
 
@@ -296,6 +611,41 @@ U
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_number_with_sign_character
+    tokens = RDoc::RubyLex.tokenize "+3--3r", nil
+
+    expected = [
+      @TK::TkINTEGER .new(0, 1, 0, "+3"),
+      @TK::TkMINUS   .new(2, 1, 2, "-"),
+      @TK::TkRATIONAL.new(3, 1, 3, "-3r"),
+      @TK::TkNL      .new(6, 1, 6, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_regexp_continuing_backslash
+    tokens = RDoc::RubyLex.tokenize "/(?<!\\\\)\\n\z/", nil
+
+    expected = [
+      @TK::TkREGEXP.new( 0, 1,  0, "/(?<!\\\\)\\n\z/"),
+      @TK::TkNL    .new(12, 1, 12, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_single_quote_escape
+    tokens = RDoc::RubyLex.tokenize %q{'\\\\ \\' \\&'}, nil
+
+    expected = [
+      @TK::TkSTRING.new( 0, 1,  0, %q{'\\\\ \\' \\&'}),
+      @TK::TkNL    .new(10, 1, 10, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_string
     tokens = RDoc::RubyLex.tokenize "'hi'", nil
 
@@ -307,6 +657,184 @@ U
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_string_with_escape
+    tokens = RDoc::RubyLex.tokenize <<'RUBY', nil
+[
+  '\\',
+  '\'',
+  "'",
+  "\'\"\`",
+  "\#",
+  "\#{}",
+  "#",
+  "#{}",
+  /'"/,
+  /\'\"/,
+  /\//,
+  /\\/,
+  /\#/,
+  /\#{}/,
+  /#/,
+  /#{}/
+]
+RUBY
+
+    expected = [
+      @TK::TkLBRACK .new(  0,  1,   0, "["),
+      @TK::TkNL     .new(  1,  1,   1, "\n"),
+      @TK::TkSPACE  .new(  2,  2,   0, "  "),
+      @TK::TkSTRING .new(  4,  2,   2, "'\\\\'"),
+      @TK::TkCOMMA  .new(  8,  2,   6, ","),
+      @TK::TkNL     .new(  9,  2,   2, "\n"),
+      @TK::TkSPACE  .new( 10,  3,   0, "  "),
+      @TK::TkSTRING .new( 12,  3,   2, "'\\''"),
+      @TK::TkCOMMA  .new( 16,  3,   6, ","),
+      @TK::TkNL     .new( 17,  3,  10, "\n"),
+      @TK::TkSPACE  .new( 18,  4,   0, "  "),
+      @TK::TkSTRING .new( 20,  4,   2, "\"'\""),
+      @TK::TkCOMMA  .new( 23,  4,   5, ","),
+      @TK::TkNL     .new( 24,  4,  18, "\n"),
+      @TK::TkSPACE  .new( 25,  5,   0, "  "),
+      @TK::TkSTRING .new( 27,  5,   2, "\"\\'\\\"\\`\""),
+      @TK::TkCOMMA  .new( 35,  5,  10, ","),
+      @TK::TkNL     .new( 36,  5,  25, "\n"),
+      @TK::TkSPACE  .new( 37,  6,   0, "  "),
+      @TK::TkSTRING .new( 39,  6,   2, "\"\\#\""),
+      @TK::TkCOMMA  .new( 43,  6,   6, ","),
+      @TK::TkNL     .new( 44,  6,  37, "\n"),
+      @TK::TkSPACE  .new( 45,  7,   0, "  "),
+      @TK::TkSTRING .new( 47,  7,   2, "\"\\\#{}\""),
+      @TK::TkCOMMA  .new( 53,  7,   8, ","),
+      @TK::TkNL     .new( 54,  7,  45, "\n"),
+      @TK::TkSPACE  .new( 55,  8,   0, "  "),
+      @TK::TkSTRING .new( 57,  8,   2, "\"#\""),
+      @TK::TkCOMMA  .new( 60,  8,   5, ","),
+      @TK::TkNL     .new( 61,  8,  55, "\n"),
+      @TK::TkSPACE  .new( 62,  9,   0, "  "),
+      @TK::TkDSTRING.new( 64,  9,   2, "\"\#{}\""),
+      @TK::TkCOMMA  .new( 69,  9,   7, ","),
+      @TK::TkNL     .new( 70,  9,  62, "\n"),
+      @TK::TkSPACE  .new( 71, 10,   0, "  "),
+      @TK::TkREGEXP .new( 73, 10,   2, "/'\"/"),
+      @TK::TkCOMMA  .new( 77, 10,   6, ","),
+      @TK::TkNL     .new( 78, 10,  71, "\n"),
+      @TK::TkSPACE  .new( 79, 11,   0, "  "),
+      @TK::TkREGEXP .new( 81, 11,   2, "/\\'\\\"/"),
+      @TK::TkCOMMA  .new( 87, 11,   8, ","),
+      @TK::TkNL     .new( 88, 11,  79, "\n"),
+      @TK::TkSPACE  .new( 89, 12,   0, "  "),
+      @TK::TkREGEXP .new( 91, 12,   2, "/\\//"),
+      @TK::TkCOMMA  .new( 95, 12,   6, ","),
+      @TK::TkNL     .new( 96, 12,  89, "\n"),
+      @TK::TkSPACE  .new( 97, 13,   0, "  "),
+      @TK::TkREGEXP .new( 99, 13,   2, "/\\\\/"),
+      @TK::TkCOMMA  .new(103, 13,   6, ","),
+      @TK::TkNL     .new(104, 13,  97, "\n"),
+      @TK::TkSPACE  .new(105, 14,   0, "  "),
+      @TK::TkREGEXP .new(107, 14,   2, "/\\#/"),
+      @TK::TkCOMMA  .new(111, 14,   6, ","),
+      @TK::TkNL     .new(112, 14, 105, "\n"),
+      @TK::TkSPACE  .new(113, 15,   0, "  "),
+      @TK::TkREGEXP .new(115, 15,   2, "/\\\#{}/"),
+      @TK::TkCOMMA  .new(121, 15,   8, ","),
+      @TK::TkNL     .new(122, 15, 113, "\n"),
+      @TK::TkSPACE  .new(123, 16,   0, "  "),
+      @TK::TkREGEXP .new(125, 16,   2, "/#/"),
+      @TK::TkCOMMA  .new(128, 16,   5, ","),
+      @TK::TkNL     .new(129, 16, 123, "\n"),
+      @TK::TkSPACE  .new(130, 17,   0, "  "),
+      @TK::TkDREGEXP.new(132, 17,   2, "/\#{}/"),
+      @TK::TkNL     .new(137, 17,   7, "\n"),
+      @TK::TkRBRACK .new(138, 18,   0, "]"),
+      @TK::TkNL     .new(139, 18, 138, "\n")
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_postfix_if_after_escaped_newline
+    tokens = RDoc::RubyLex.tokenize <<'RUBY', nil
+def a
+  1 if true
+  1 \
+    if true
+end
+RUBY
+
+    expected = [
+      @TK::TkDEF       .new( 0, 1, 0,  "def"),
+      @TK::TkSPACE     .new( 3, 1, 3,  " "),
+      @TK::TkIDENTIFIER.new( 4, 1, 4,  "a"),
+      @TK::TkNL        .new( 5, 1, 5,  "\n"),
+      @TK::TkSPACE     .new( 6, 2, 0,  "  "),
+      @TK::TkINTEGER   .new( 8, 2, 2,  "1"),
+      @TK::TkSPACE     .new( 9, 2, 3,  " "),
+      @TK::TkIF_MOD    .new(10, 2, 4,  "if"),
+      @TK::TkSPACE     .new(12, 2, 6,  " "),
+      @TK::TkTRUE      .new(13, 2, 7,  "true"),
+      @TK::TkNL        .new(17, 2, 6,  "\n"),
+      @TK::TkSPACE     .new(18, 3, 0,  "  "),
+      @TK::TkINTEGER   .new(20, 3, 2,  "1"),
+      @TK::TkSPACE     .new(21, 3, 3,  " "),
+      @TK::TkBACKSLASH .new(22, 3, 4,  "\\"),
+      @TK::TkNL        .new(23, 3, 18, "\n"),
+      @TK::TkSPACE     .new(24, 4, 0,  "    "),
+      @TK::TkIF_MOD    .new(28, 4, 4,  "if"),
+      @TK::TkSPACE     .new(30, 4, 6,  " "),
+      @TK::TkTRUE      .new(31, 4, 7,  "true"),
+      @TK::TkNL        .new(35, 4, 24, "\n"),
+      @TK::TkEND       .new(36, 5, 0,  "end"),
+      @TK::TkNL        .new(39, 5, 36, "\n")
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_backtick_with_escape
+    tokens = RDoc::RubyLex.tokenize <<'RUBY', nil
+[
+  `\\`,
+  `\'\"\``,
+  `\#`,
+  `\#{}`,
+  `#`,
+  `#{}`
+]
+RUBY
+
+    expected = [
+      @TK::TkLBRACK  .new( 0, 1,  0, "["),
+      @TK::TkNL      .new( 1, 1,  1, "\n"),
+      @TK::TkSPACE   .new( 2, 2,  0, "  "),
+      @TK::TkXSTRING .new( 4, 2,  2, "`\\\\`"),
+      @TK::TkCOMMA   .new( 8, 2,  6, ","),
+      @TK::TkNL      .new( 9, 2,  2, "\n"),
+      @TK::TkSPACE   .new(10, 3,  0, "  "),
+      @TK::TkXSTRING .new(12, 3,  2, "`\\'\\\"\\``"),
+      @TK::TkCOMMA   .new(20, 3, 10, ","),
+      @TK::TkNL      .new(21, 3, 10, "\n"),
+      @TK::TkSPACE   .new(22, 4,  0, "  "),
+      @TK::TkXSTRING .new(24, 4,  2, "`\\#`"),
+      @TK::TkCOMMA   .new(28, 4,  6, ","),
+      @TK::TkNL      .new(29, 4, 22, "\n"),
+      @TK::TkSPACE   .new(30, 5,  0, "  "),
+      @TK::TkXSTRING .new(32, 5,  2, "`\\\#{}`"),
+      @TK::TkCOMMA   .new(38, 5,  8, ","),
+      @TK::TkNL      .new(39, 5, 30, "\n"),
+      @TK::TkSPACE   .new(40, 6,  0, "  "),
+      @TK::TkXSTRING .new(42, 6,  2, "`#`"),
+      @TK::TkCOMMA   .new(45, 6,  5, ","),
+      @TK::TkNL      .new(46, 6, 40, "\n"),
+      @TK::TkSPACE   .new(47, 7,  0, "  "),
+      @TK::TkDXSTRING.new(49, 7,  2, "`\#{}`"),
+      @TK::TkNL      .new(54, 7,  7, "\n"),
+      @TK::TkRBRACK  .new(55, 8,  0, "]"),
+      @TK::TkNL      .new(56, 8, 55, "\n")
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_string_escape
     tokens = RDoc::RubyLex.tokenize '"\\n"', nil
     assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\n\""), tokens.first
@@ -390,17 +918,87 @@ U
     expected = [
       @TK::TkIDENTIFIER.new( 0, 1,  0, 'scope'),
       @TK::TkSPACE     .new( 5, 1,  5, ' '),
-      @TK::TkIDENTIFIER.new( 6, 1,  6, 'module'),
+      @TK::TkSYMBOL    .new( 6, 1,  6, 'module:'),
-      @TK::TkCOLON     .new(12, 1, 12, ':'),
       @TK::TkSPACE     .new(13, 1, 13, ' '),
-      @TK::TkSYMBEG    .new(14, 1, 14, ':'),
+      @TK::TkSYMBOL    .new(14, 1, 14, ':v1'),
-      @TK::TkIDENTIFIER.new(15, 1, 15, 'v1'),
       @TK::TkNL        .new(17, 1, 17, "\n"),
     ]
 
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_particular_kind_of_symbols
+    tokens = RDoc::RubyLex.tokenize '{ Thomas: :Thomas, Dave!: :Dave!, undef: :undef }', nil
+
+    expected = [
+      @TK::TkLBRACE.new( 0, 1,  0, "{"),
+      @TK::TkSPACE .new( 1, 1,  1, " "),
+      @TK::TkSYMBOL.new( 2, 1,  2, "Thomas:"),
+      @TK::TkSPACE .new( 9, 1,  9, " "),
+      @TK::TkSYMBOL.new(10, 1, 10, ":Thomas"),
+      @TK::TkCOMMA .new(17, 1, 17, ","),
+      @TK::TkSPACE .new(18, 1, 18, " "),
+      @TK::TkSYMBOL.new(19, 1, 19, "Dave!:"),
+      @TK::TkSPACE .new(25, 1, 25, " "),
+      @TK::TkSYMBOL.new(26, 1, 26, ":Dave!"),
+      @TK::TkCOMMA .new(32, 1, 32, ","),
+      @TK::TkSPACE .new(33, 1, 33, " "),
+      @TK::TkSYMBOL.new(34, 1, 34, "undef:"),
+      @TK::TkSPACE .new(40, 1, 40, " "),
+      @TK::TkSYMBOL.new(41, 1, 41, ":undef"),
+      @TK::TkSPACE .new(47, 1, 47, " "),
+      @TK::TkRBRACE.new(48, 1, 48, "}"),
+      @TK::TkNL    .new(49, 1, 49, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_symbol_for_nested_method
+    tokens = RDoc::RubyLex.tokenize 'return untrace_var :name', nil
+
+    expected = [
+      @TK::TkRETURN    .new( 0, 1,  0, "return"),
+      @TK::TkSPACE     .new( 6, 1,  6, " "),
+      @TK::TkIDENTIFIER.new( 7, 1,  7, "untrace_var"),
+      @TK::TkSPACE     .new(18, 1, 18, " "),
+      @TK::TkSYMBOL    .new(19, 1, 19, ":name"),
+      @TK::TkNL        .new(24, 1, 24, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_symbol_with_quote
+    tokens = RDoc::RubyLex.tokenize <<RUBY, nil
+a.include?()?"a":"b"
+{"t":1,'t2':2}
+RUBY
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1,  0, "a"),
+      @TK::TkDOT       .new( 1, 1,  1, "."),
+      @TK::TkFID       .new( 2, 1,  2, "include?"),
+      @TK::TkLPAREN    .new(10, 1, 10, "("),
+      @TK::TkRPAREN    .new(11, 1, 11, ")"),
+      @TK::TkQUESTION  .new(12, 1, 12, "?"),
+      @TK::TkSTRING    .new(13, 1, 13, "\"a\""),
+      @TK::TkCOLON     .new(16, 1, 16, ":"),
+      @TK::TkSTRING    .new(17, 1, 17, "\"b\""),
+      @TK::TkNL        .new(20, 1, 20, "\n"),
+      @TK::TkLBRACE    .new(21, 2,  0, "{"),
+      @TK::TkSYMBOL    .new(22, 2,  1, "\"t\":"),
+      @TK::TkINTEGER   .new(26, 2,  5, "1"),
+      @TK::TkCOMMA     .new(27, 2,  6, ","),
+      @TK::TkSYMBOL    .new(28, 2,  7, "'t2':"),
+      @TK::TkINTEGER   .new(33, 2, 12, "2"),
+      @TK::TkRBRACE    .new(34, 2, 13, "}"),
+      @TK::TkNL        .new(35, 2, 21, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_unary_minus
     ruby_lex = RDoc::RubyLex.new("-1", nil)
     assert_equal("-1", ruby_lex.token.value)
@@ -418,5 +1016,80 @@ U
     assert_equal("-0.1", ruby_lex.token.value)
   end
 
+  def test_rational_imaginary_tokenize
+    tokens = RDoc::RubyLex.tokenize '1.11r + 2.34i + 5.55ri + 0i', nil
+
+    expected = [
+      @TK::TkRATIONAL .new( 0, 1,  0, '1.11r'),
+      @TK::TkSPACE    .new( 5, 1,  5, ' '),
+      @TK::TkPLUS     .new( 6, 1,  6, '+'),
+      @TK::TkSPACE    .new( 7, 1,  7, ' '),
+      @TK::TkIMAGINARY.new( 8, 1,  8, '2.34i'),
+      @TK::TkSPACE    .new(13, 1, 13, ' '),
+      @TK::TkPLUS     .new(14, 1, 14, '+'),
+      @TK::TkSPACE    .new(15, 1, 15, ' '),
+      @TK::TkIMAGINARY.new(16, 1, 16, '5.55ri'),
+      @TK::TkSPACE    .new(22, 1, 22, ' '),
+      @TK::TkPLUS     .new(23, 1, 23, '+'),
+      @TK::TkSPACE    .new(24, 1, 24, ' '),
+      @TK::TkIMAGINARY.new(25, 1, 25, '0i'),
+      @TK::TkNL       .new(27, 1, 27, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_square_bracket_as_method
+    tokens = RDoc::RubyLex.tokenize "Array.[](1, 2)", nil
+
+    expected = [
+      @TK::TkCONSTANT  .new(0,  1,  0, "Array"),
+      @TK::TkDOT       .new(5,  1,  5, "."),
+      @TK::TkIDENTIFIER.new(6,  1,  6, "[]"),
+      @TK::TkfLPAREN   .new(8,  1,  8, "("),
+      @TK::TkINTEGER   .new(9,  1,  9, "1"),
+      @TK::TkCOMMA     .new(10, 1, 10, ","),
+      @TK::TkSPACE     .new(11, 1, 11, " "),
+      @TK::TkINTEGER   .new(12, 1, 12, "2"),
+      @TK::TkRPAREN    .new(13, 1, 13, ")"),
+      @TK::TkNL        .new(14, 1, 14, "\n")
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_constant_with_exclamation
+    tokens = RDoc::RubyLex.tokenize "Hello there, Dave!", nil
+
+    expected = [
+      @TK::TkCONSTANT  .new( 0, 1,  0, "Hello"),
+      @TK::TkSPACE     .new( 5, 1,  5, " "),
+      @TK::TkIDENTIFIER.new( 6, 1,  6, "there"),
+      @TK::TkCOMMA     .new(11, 1, 11, ","),
+      @TK::TkSPACE     .new(12, 1, 12, " "),
+      @TK::TkIDENTIFIER.new(13, 1, 13, "Dave!"),
+      @TK::TkNL        .new(18, 1, 18, "\n")
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_identifer_not_equal
+    tokens = RDoc::RubyLex.tokenize "foo!=bar\nfoo?=bar", nil
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1, 0, "foo"),
+      @TK::TkNEQ       .new( 3, 1, 3, "!="),
+      @TK::TkIDENTIFIER.new( 5, 1, 5, "bar"),
+      @TK::TkNL        .new( 8, 1, 8, "\n"),
+      @TK::TkFID       .new( 9, 2, 0, "foo?"),
+      @TK::TkASSIGN    .new(13, 2, 4, "="),
+      @TK::TkIDENTIFIER.new(14, 2, 5, "bar"),
+      @TK::TkNL        .new(17, 2, 9, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
 end
 

test/rdoc/test_rdoc_store.rb

@@ -162,7 +162,7 @@ class TestRDocStore < XrefTestCase
 
   def test_all_classes_and_modules
     expected = %w[
-      C1 C2 C2::C3 C2::C3::H1 C3 C3::H1 C3::H2 C4 C4::C4 C5 C5::C1 C6
+      C1 C2 C2::C3 C2::C3::H1 C3 C3::H1 C3::H2 C4 C4::C4 C5 C5::C1 C6 C7
       Child
       M1 M1::M2
       Parent
@@ -213,7 +213,7 @@ class TestRDocStore < XrefTestCase
 
   def test_classes
     expected = %w[
-      C1 C2 C2::C3 C2::C3::H1 C3 C3::H1 C3::H2 C4 C4::C4 C5 C5::C1 C6
+      C1 C2 C2::C3 C2::C3::H1 C3 C3::H1 C3::H2 C4 C4::C4 C5 C5::C1 C6 C7
       Child
       Parent
     ]

test/rdoc/xref_data.rb

@@ -82,6 +82,18 @@ class C6
   def prot6() end
 end
 
+class C7
+  attr_reader :attr_reader
+  attr_reader :attr_reader_nodoc # :nodoc:
+  attr_writer :attr_writer
+  attr_writer :attr_writer_nodoc # :nodoc:
+  attr_accessor :attr_accessor
+  attr_accessor :attr_accessor_nodoc # :nodoc:
+
+  CONST = :const
+  CONST_NODOC = :const_nodoc # :nodoc:
+end
+
 module M1
   def m
   end

test/rdoc/xref_test_case.rb

@@ -52,6 +52,7 @@ class XrefTestCase < RDoc::TestCase
     @c3_h1 = @xref_data.find_module_named 'C3::H1'
     @c3_h2 = @xref_data.find_module_named 'C3::H2'
     @c6    = @xref_data.find_module_named 'C6'
+    @c7    = @xref_data.find_module_named 'C7'
 
     @m1    = @xref_data.find_module_named 'M1'
     @m1_m  = @m1.method_list.first

tool/sync_default_gems.rb

@@ -1,7 +1,7 @@
 # sync following repositories to ruby repository
 #
 # * https://github.com/rubygems/rubygems
-# * https://github.com/rdoc/rdoc
+# * https://github.com/ruby/rdoc
 # * https://github.com/flori/json
 # * https://github.com/ruby/psych
 # * https://github.com/ruby/fileutils
@@ -25,7 +25,7 @@
 
 $repositories = {
   rubygems: 'rubygems/rubygems',
-  rdoc: 'rdoc/rdoc',
+  rdoc: 'ruby/rdoc',
   json: 'flori/json',
   psych: 'ruby/psych',
   fileutils: 'ruby/fileutils',
@@ -65,10 +65,11 @@ def sync_default_gems(gem)
     `cp ../../rubygems/rubygems/LICENSE.txt ./lib/rubygems`
   when "rdoc"
     `rm -rf lib/rdoc* test/rdoc`
-    `cp -rf ../../rdoc/rdoc/lib/rdoc* ./lib`
+    `cp -rf ../rdoc/lib/rdoc* ./lib`
-    `cp -rf ../../rdoc/rdoc/test test/rdoc`
+    `cp -rf ../rdoc/test test/rdoc`
-    `cp ../../rdoc/rdoc/rdoc.gemspec ./lib/rdoc`
+    `cp ../rdoc/rdoc.gemspec ./lib/rdoc`
     `rm -f lib/rdoc/markdown.kpeg lib/rdoc/markdown/literals.kpeg lib/rdoc/rd/block_parser.ry lib/rdoc/rd/inline_parser.ry`
+    `git checkout lib/rdoc/.document`
   when "json"
     `rm -rf ext/json test/json`
     `cp -rf ../../flori/json/ext/json/ext ext/json`