* parse.y (parser_yylex): should clear parser->tokp as well.

[ruby-dev:32250]

* parse.y: remove NEED_ASSOC that break test_parser_events.

* parse.y (parser_yylex): should not decrement line numbers at the
  end of file.

* file.c (rb_find_file_ext): search .rb files first through in the
  loadpath.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@13966 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -48,6 +48,19 @@ Sat Nov 17 23:51:29 2007  Masaki Suketa  <masaki.suketa@nifty.ne.jp>
 
 	* ext/win32ole/win32ole.c (ole_invoke): bug fix. [ruby-talk:279100]
 
+Sat Nov 17 23:21:15 2007  Yukihiro Matsumoto  <matz@ruby-lang.org>
+
+	* parse.y (parser_yylex): should clear parser->tokp as well.
+	  [ruby-dev:32250]
+
+	* parse.y: remove NEED_ASSOC that break test_parser_events.
+
+	* parse.y (parser_yylex): should not decrement line numbers at the
+	  end of file.
+
+	* file.c (rb_find_file_ext): search .rb files first through in the
+	  loadpath.
+
 Fri Nov 16 23:31:18 2007  Yukihiro Matsumoto  <matz@ruby-lang.org>
 
 	* bignum.c (rb_big_odd_p): new method added.  a patch from Tadashi

ext/ripper/lib/ripper/lexer.rb

@@ -23,7 +23,7 @@ class Ripper
   #   require 'ripper'
   #   require 'pp'
   #
-  #   p Ripper.scan("def m(a) nil end")
+  #   p Ripper.lex("def m(a) nil end")
   #     #=> [[[1,  0], :on_kw,     "def"],
   #          [[1,  3], :on_sp,     " "  ],
   #          [[1,  4], :on_ident,  "m"  ],

file.c

@@ -4197,16 +4197,16 @@ rb_find_file_ext(VALUE *filep, const char *const *ext)
     if (!rb_load_path) return 0;
 
     Check_Type(rb_load_path, T_ARRAY);
-    for (i=0;i<RARRAY_LEN(rb_load_path);i++) {
+    for (j=0; ext[j]; j++) {
-	VALUE str = RARRAY_PTR(rb_load_path)[i];
+	fname = rb_str_dup(*filep);
+	rb_str_cat2(fname, ext[j]);
+	OBJ_FREEZE(fname);
+	for (i=0;i<RARRAY_LEN(rb_load_path);i++) {
+	    VALUE str = RARRAY_PTR(rb_load_path)[i];
 
-	FilePathValue(str);
+	    FilePathValue(str);
-	if (RSTRING_LEN(str) == 0) continue;
+	    if (RSTRING_LEN(str) == 0) continue;
-	path = RSTRING_PTR(str);
+	    path = RSTRING_PTR(str);
-	for (j=0; ext[j]; j++) {
-	    fname = rb_str_dup(*filep);
-	    rb_str_cat2(fname, ext[j]);
-	    OBJ_FREEZE(fname);
 	    found = dln_find_file(StringValueCStr(fname), path);
 	    if (found && file_load_ok(found)) {
 		*filep = rb_str_new2(found);

parse.y

@@ -565,15 +565,6 @@ static void ripper_compile_error(struct parser_params*, const char *fmt, ...);
 # define PARSER_ARG ruby_sourcefile, ruby_sourceline,
 #endif
 
-#ifdef RIPPER
-#define NEED_ASSOC(cons, car, cdr) do {			      \
-	if ((cons) == (car) || (cons) == (cdr) ||	      \
-	    TYPE(cons) != T_ARRAY || RARRAY_LEN(cons) != 2) { \
-	    (cons) = rb_assoc_new((car), (cdr));	      \
-	}						      \
-    } while (0)
-#endif
-
 /* Older versions of Yacc set YYMAXDEPTH to a very low value by default (150,
    for instance).  This is too low for Ruby to parse some files, such as
    date/format.rb, therefore bump the value up to at least Bison's default. */
@@ -4413,7 +4404,6 @@ assoc		: arg_value tASSOC arg_value
 			$$ = list_append(NEW_LIST($1), $3);
 		    /*%
 			$$ = dispatch2(assoc_new, $1, $3);
-			NEED_ASSOC($$, $1, $3);
 		    %*/
 		    }
 		| tLABEL arg_value
@@ -4422,7 +4412,6 @@ assoc		: arg_value tASSOC arg_value
 			$$ = list_append(NEW_LIST(NEW_LIT(ID2SYM($1))), $2);
 		    /*%
 			$$ = dispatch2(assoc_new, $1, $2);
-			NEED_ASSOC($$, $1, $2);
 		    %*/
 		    }
 		;
@@ -6003,21 +5992,6 @@ parser_yylex(struct parser_params *parser)
     cmd_state = command_start;
     command_start = Qfalse;
   retry:
-#ifdef RIPPER
-    while ((c = nextc())) {
-        switch (c) {
-          case ' ': case '\t': case '\f': case '\r':
-          case '\13': /* '\v' */
-            space_seen++;
-            break;
-          default:
-            goto outofloop;
-        }
-    }
-  outofloop:
-    pushback(c);
-    ripper_dispatch_scan_event(parser, tSP);
-#endif
     switch (c = nextc()) {
       case '\0':		/* NUL */
       case '\004':		/* ^D */
@@ -6029,6 +6003,20 @@ parser_yylex(struct parser_params *parser)
       case ' ': case '\t': case '\f': case '\r':
       case '\13': /* '\v' */
 	space_seen++;
+#ifdef RIPPER
+	while ((c = nextc())) {
+	    switch (c) {
+	      case ' ': case '\t': case '\f': case '\r':
+	      case '\13': /* '\v' */
+		break;
+	      default:
+		goto outofloop;
+	    }
+	}
+      outofloop:
+	pushback(c);
+	ripper_dispatch_scan_event(parser, tSP);
+#endif
 	goto retry;
 
       case '#':		/* it's a comment */
@@ -6070,16 +6058,22 @@ parser_yylex(struct parser_params *parser)
 		space_seen++;
 		break;
 	      case '.': {
-		if ((c = nextc()) != '.') {
+		  if ((c = nextc()) != '.') {
-		    pushback(c);
+		      pushback(c);
-		    pushback('.');
+		      pushback('.');
 		    goto retry;
-		}
+		  }
 	      }
 	      default:
+		--ruby_sourceline;
+	      case -1:		/* EOF no decrement*/
 		lex_nextline = lex_lastline;
 		lex_p = lex_pend;
-		--ruby_sourceline;
+#ifdef RIPPER
+		if (c != -1) {
+		    parser->tokp = lex_p;
+		}
+#endif
 		goto normal_newline;
 	    }
 	}

test/ripper/dummyparser.rb

@@ -30,7 +30,7 @@ class NodeList
   end
 
   def prepend(items)
-    @list[0,0] = items
+    @list.unshift items
   end
 
   def to_s
@@ -98,32 +98,44 @@ class DummyParser < Ripper
     args
   end
 
-  def on_arglist_new
+  def on_args_new
     NodeList.new
   end
 
-  def on_arglist_add(list, arg)
+  def on_args_add(list, arg)
     list.push(arg)
   end
 
-  def on_arglist_add_block(list, blk)
+  def on_args_add_block(list, blk)
-    list.push('&' + blk.to_s)
+    if blk
+      list.push('&' + blk.to_s)
+    else
+      list
+    end
   end
 
-  def on_arglist_add_star(list, arg)
+  def on_args_add_star(list, arg)
     list.push('*' + arg.to_s)
   end
 
-  def on_arglist_prepend(list, args)
+  def on_args_prepend(list, args)
     list.prepend args
     list
   end
 
   def on_method_add_arg(m, arg)
+    if arg == nil
+      arg = on_args_new
+    end
     m.children.push arg
     m
   end
 
+  def on_method_add_block(m, b)
+    on_args_add_block(m.children, b)
+    m
+  end
+
   def on_assoc_new(a, b)
     Node.new('assoc', a, b)
   end
@@ -390,7 +402,7 @@ class DummyParser < Ripper
     Node.new('sclass', a, b)
   end
 
-  def on_space(a)
+  def on_sp(a)
     Node.new('space', a)
   end
 
@@ -514,4 +526,46 @@ class DummyParser < Ripper
     Node.new('zsuper')
   end
 
+  def on_backref(a)
+    a
+  end
+  def on_comma(a)
+    a
+  end
+  def on_gvar(a)
+    a
+  end
+  def on_ident(a)
+    a
+  end
+  def on_int(a)
+    a
+  end
+  def on_kw(a)
+    a
+  end
+  def on_lbrace(a)
+    a
+  end
+  def on_rbrace(a)
+    a
+  end
+  def on_lbracket(a)
+    a
+  end
+  def on_rbracket(a)
+    a
+  end
+  def on_lparen(a)
+    a
+  end
+  def on_rparen(a)
+    a
+  end
+  def on_op(a)
+    a
+  end
+  def on_semicolon(a)
+    a
+  end
 end

test/ripper/test_files.rb

@@ -22,4 +22,4 @@ end
 
 rescue LoadError
 end
-  
+  

test/ripper/test_parser_events.rb

@@ -495,4 +495,4 @@ class TestRipper_ParserEvents < Test::Unit::TestCase
 end
 
 rescue LoadError
-end
+end