* ext/ripper: refactoring code generation tools. [ruby-dev:27247] [ruby-dev:27273]

* ext/ripper/depend: use generate.rb.
* ext/ripper/lib/ripper/core.rb: removed.
* ext/ripper/tools/generate-eventids1.rb: removed (code moved to generate.rb).
* ext/ripper/tools/generate-ripper_rb.rb: removed (code moved to generate.rb).
* ext/ripper/tools/list-parse-event-ids.rb: removed (code moved to generate.rb).
* ext/ripper/tools/list-scan-event-ids.rb: removed (code moved to generate.rb).
* ext/ripper/lib/ripper/core.rb: removed.
* ext/ripper: refactoring tests. [ruby-dev:27273]
* ext/ripper/test/check-event-arity.rb: removed (code moved to tools/generate.rb).
* ext/ripper/test/check-event-coverage.rb: removed (code moved to test/ripper/test_parser_events.rb).
* ext/ripper/test/check-scanner-event-coverage.rb: removed (code moved to test/ripper/test_scanner_events.rb).
* ext/ripper/test/list-called-events.rb: removed.
* ext/ripper/test/src_rb: removed.
* ext/ripper/test/validate.rb: removed.
* test/ripper/test_scanner_events.rb: check event coverage.
* ext/ripper/lib/ripper/core.rb.in: update copyright year.


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@9273 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,47 @@
+Fri Sep 23 05:57:23 2005  Minero Aoki  <aamine@loveruby.net>
+
+	* ext/ripper: refactoring code generation tools. [ruby-dev:27247]
+	  [ruby-dev:27273]
+
+	* ext/ripper/depend: use generate.rb.
+
+	* ext/ripper/lib/ripper/core.rb: removed.
+
+	* ext/ripper/tools/generate-eventids1.rb: removed (code moved to
+	  generate.rb).
+
+	* ext/ripper/tools/generate-ripper_rb.rb: removed (code moved to
+	  generate.rb).
+
+	* ext/ripper/tools/list-parse-event-ids.rb: removed (code moved to
+	  generate.rb).
+
+	* ext/ripper/tools/list-scan-event-ids.rb: removed (code moved to
+	  generate.rb).
+
+	* ext/ripper/lib/ripper/core.rb: removed.
+
+	* ext/ripper: refactoring tests. [ruby-dev:27273]
+
+	* ext/ripper/test/check-event-arity.rb: removed (code moved to
+	  tools/generate.rb).
+
+	* ext/ripper/test/check-event-coverage.rb: removed (code moved to
+	  test/ripper/test_parser_events.rb).
+
+	* ext/ripper/test/check-scanner-event-coverage.rb: removed (code
+	  moved to test/ripper/test_scanner_events.rb).
+
+	* ext/ripper/test/list-called-events.rb: removed.
+
+	* ext/ripper/test/src_rb: removed.
+
+	* ext/ripper/test/validate.rb: removed.
+
+	* test/ripper/test_scanner_events.rb: check event coverage.
+
+	* ext/ripper/lib/ripper/core.rb.in: update copyright year.
+
 Thu Sep 22 23:40:19 2005  Nobuyoshi Nakada  <nobu@ruby-lang.org>
 
 	* lib/mkmf.rb (find_executable0): default path if environment is not

ext/ripper/README

@@ -13,13 +13,12 @@ Requirements
 ------------
 
   * ruby 1.9 (support CVS HEAD only)
-  * bison 1.28 or later (You MUST use bison)
+  * bison 1.28 or later (Other yaccs do not work)
 
 Usage
 -----
 
   See test/ripper/* and sample/ripper/*.
-  To know thrown events, see lib/ripper.rb.
 
 License
 -------

ext/ripper/depend

@@ -1,39 +1,18 @@
-c : ripper.c eventids1.c
+ripper.o: ripper.c eventids1.c $(srcdir)/eventids2.c $(hdrdir)/lex.c
-dev: c rb
-
-ripper.o: ripper.c $(hdrdir)/lex.c eventids1.c $(srcdir)/eventids2.c
 
 .y.c:
 	bison -t -v -o$@ $<
 
 ripper.y: $(hdrdir)/parse.y $(srcdir)/tools/preproc.rb
-	$(RUBY) $(srcdir)/tools/preproc.rb $(hdrdir)/parse.y > $@
+	$(RUBY) $(srcdir)/tools/preproc.rb $(hdrdir)/parse.y $@
 
-eventids1.c: $(hdrdir)/parse.y $(srcdir)/tools/generate-eventids1.rb ids1
+$(srcdir)/lib/ripper/core.rb: $(srcdir)/lib/ripper/core.rb.in $(srcdir)/tools/generate.rb
-	$(RUBY) $(srcdir)/tools/generate-eventids1.rb ids1 > $@
+	$(RUBY) $(srcdir)/tools/generate.rb --mode=ripper/core --template=$@.in --output=$@ --ids1src=$(hdrdir)/parse.y --ids2src=$(srcdir)/eventids2.c
 
-ids1: $(srcdir)/tools/list-parse-event-ids.rb $(hdrdir)/parse.y
+eventids1.c: $(hdrdir)/parse.y $(srcdir)/tools/generate.rb
-	$(RUBY) $(srcdir)/tools/list-parse-event-ids.rb -a $(hdrdir)/parse.y > $@
+	$(RUBY) $(srcdir)/tools/generate.rb --mode=eventids1 --ids2src=$(hdrdir)/parse.y --output=$@
 
-ids2: $(srcdir)/tools/list-scan-event-ids.rb $(srcdir)/eventids2.c
+# Entries for Ripper maintainer
-	$(RUBY) $(srcdir)/tools/list-scan-event-ids.rb -a $(srcdir)/eventids2.c > $@
-
-#
-# Entries for only ripper developpers: DO NOT USE
-#
-
-# [WARNING]
-# Never make core.rb automatically; it causes line terminator problem.
-# See [ruby-dev:26462]
-rb: $(srcdir)/lib/ripper/core.rb.in ids1 ids2 $(srcdir)/tools/generate-ripper_rb.rb
-	$(RUBY) $(srcdir)/tools/generate-ripper_rb.rb $(srcdir)/lib/ripper/core.rb.in ids1 ids2 > $(srcdir)/lib/ripper/core.rb
-
-test:
-	$(RUBY) test/check-event-arity.rb $(hdrdir)/parse.y
-	sh test/check-event-coverage.sh
-	$(RUBY) test/check-scanner-event-coverage.rb
-	$(RUBY) test/validate.rb > /dev/null
-	@echo OK
 
 preproc: ripper.E
 ripper.E: ripper.c

ext/ripper/lib/ripper/core.rb

@@ -1,913 +0,0 @@
-# This file is automatically generated from core.rb.in and parse.y.
-# DO NOT MODIFY!!!!!!
-
-#
-# ripper/core.rb
-#
-# Copyright (C) 2003,2004 Minero Aoki
-#
-# This program is free software.
-# You can distribute and/or modify this program under the Ruby License.
-# For details of Ruby License, see ruby/COPYING.
-#
-
-require 'ripper.so'
-
-class Ripper
-  # Parses Ruby program read from _src_.
-  # _src_ must be a String or a IO or a object which has #gets method.
-  def Ripper.parse(src, filename = '(ripper)', lineno = 1)
-    new(src, filename, lineno).parse
-  end
-
-  # This table contains name of parser events and its arity.
-  PARSER_EVENT_TABLE = {
-    :BEGIN => 1,
-    :END => 1,
-    :alias => 2,
-    :alias_error => 1,
-    :aref => 2,
-    :aref_field => 2,
-    :arg_ambiguous => 0,
-    :arg_paren => 1,
-    :arglist_add => 2,
-    :arglist_add_block => 2,
-    :arglist_add_star => 2,
-    :arglist_new => 0,
-    :arglist_prepend => 2,
-    :array => 1,
-    :assign => 2,
-    :assign_error => 1,
-    :assoc_new => 2,
-    :assoclist_from_args => 1,
-    :bare_assoc_hash => 1,
-    :begin => 1,
-    :binary => 3,
-    :blockvar_add_block => 2,
-    :blockvar_add_star => 2,
-    :blockvar_new => 1,
-    :bodystmt => 4,
-    :brace_block => 2,
-    :break => 1,
-    :call => 3,
-    :case => 2,
-    :class => 3,
-    :class_name_error => 1,
-    :command => 2,
-    :command_call => 4,
-    :const_ref => 1,
-    :constpath_field => 2,
-    :constpath_ref => 2,
-    :def => 3,
-    :defined => 1,
-    :defs => 5,
-    :do_block => 2,
-    :dot2 => 2,
-    :dot3 => 2,
-    :dyna_symbol => 1,
-    :else => 1,
-    :elsif => 3,
-    :ensure => 1,
-    :fcall => 1,
-    :field => 3,
-    :for => 3,
-    :hash => 1,
-    :if => 3,
-    :if_mod => 2,
-    :ifop => 3,
-    :iter_block => 2,
-    :lambda => 2,
-    :massign => 2,
-    :method_add_arg => 2,
-    :mlhs_add => 2,
-    :mlhs_add_star => 2,
-    :mlhs_new => 0,
-    :mlhs_paren => 1,
-    :module => 2,
-    :mrhs_add => 2,
-    :mrhs_add_star => 2,
-    :mrhs_new => 0,
-    :mrhs_new_from_arglist => 1,
-    :next => 1,
-    :opassign => 3,
-    :param_error => 1,
-    :params => 4,
-    :paren => 1,
-    :parse_error => 1,
-    :pragma => 2,
-    :program => 1,
-    :qwords_add => 2,
-    :qwords_new => 0,
-    :redo => 0,
-    :regexp_literal => 2,
-    :rescue => 4,
-    :rescue_mod => 2,
-    :restparam => 1,
-    :retry => 0,
-    :return => 1,
-    :return0 => 0,
-    :sclass => 2,
-    :space => 1,
-    :stmts_add => 2,
-    :stmts_new => 0,
-    :string_add => 2,
-    :string_concat => 2,
-    :string_content => 0,
-    :string_dvar => 1,
-    :string_embexpr => 1,
-    :string_literal => 1,
-    :super => 1,
-    :symbol => 1,
-    :symbol_literal => 1,
-    :topconst_field => 1,
-    :topconst_ref => 1,
-    :unary => 2,
-    :undef => 1,
-    :unless => 3,
-    :unless_mod => 2,
-    :until => 2,
-    :until_mod => 2,
-    :var_alias => 2,
-    :var_field => 1,
-    :var_ref => 1,
-    :void_stmt => 0,
-    :when => 3,
-    :while => 2,
-    :while_mod => 2,
-    :word_add => 2,
-    :word_new => 0,
-    :words_add => 2,
-    :words_new => 0,
-    :xstring_add => 2,
-    :xstring_literal => 1,
-    :xstring_new => 0,
-    :yield => 1,
-    :yield0 => 0,
-    :zsuper => 0
-  }
-
-  # This array contains name of parser events.
-  PARSER_EVENTS = PARSER_EVENT_TABLE.keys
-
-  # This table contains name of scanner events and its arity
-  # (arity is always 1 for all scanner events).
-  SCANNER_EVENT_TABLE = {
-    :CHAR => 1,
-    :__end__ => 1,
-    :backref => 1,
-    :backtick => 1,
-    :comma => 1,
-    :comment => 1,
-    :const => 1,
-    :cvar => 1,
-    :embdoc => 1,
-    :embdoc_beg => 1,
-    :embdoc_end => 1,
-    :embexpr_beg => 1,
-    :embexpr_end => 1,
-    :embvar => 1,
-    :float => 1,
-    :gvar => 1,
-    :heredoc_beg => 1,
-    :heredoc_end => 1,
-    :ident => 1,
-    :ignored_nl => 1,
-    :int => 1,
-    :ivar => 1,
-    :kw => 1,
-    :label => 1,
-    :lbrace => 1,
-    :lbracket => 1,
-    :lparen => 1,
-    :nl => 1,
-    :op => 1,
-    :period => 1,
-    :qwords_beg => 1,
-    :rbrace => 1,
-    :rbracket => 1,
-    :regexp_beg => 1,
-    :regexp_end => 1,
-    :rparen => 1,
-    :semicolon => 1,
-    :sp => 1,
-    :symbeg => 1,
-    :tlambda => 1,
-    :tlambda_arg => 1,
-    :tlambeg => 1,
-    :tstring_beg => 1,
-    :tstring_content => 1,
-    :tstring_end => 1,
-    :words_beg => 1,
-    :words_sep => 1
-  }
-
-  # This array contains name of scanner events.
-  SCANNER_EVENTS = SCANNER_EVENT_TABLE.keys
-
-  # This table contains name of all ripper events.
-  EVENTS = PARSER_EVENTS + SCANNER_EVENTS
-
-  ###                ###
-  ### Event Handlers ###
-  ###                ###
-
-  private
-
-  # This method is called when weak warning is produced by the parser.
-  # _fmt_ and _args_ is printf style.
-  def warn(fmt, *args)
-  end
-
-  # This method is called when strong warning is produced by the parser.
-  # _fmt_ and _args_ is printf style.
-  def warning(fmt, *args)
-  end
-
-  # This method is called when the parser found syntax error.
-  def compile_error(msg)
-  end
-
-  #
-  # Parser Events
-  #
-
-  def on_BEGIN(a)
-    a
-  end
-
-  def on_END(a)
-    a
-  end
-
-  def on_alias(a, b)
-    a
-  end
-
-  def on_alias_error(a)
-    a
-  end
-
-  def on_aref(a, b)
-    a
-  end
-
-  def on_aref_field(a, b)
-    a
-  end
-
-  def on_arg_ambiguous
-    nil
-  end
-
-  def on_arg_paren(a)
-    a
-  end
-
-  def on_arglist_add(a, b)
-    a
-  end
-
-  def on_arglist_add_block(a, b)
-    a
-  end
-
-  def on_arglist_add_star(a, b)
-    a
-  end
-
-  def on_arglist_new
-    nil
-  end
-
-  def on_arglist_prepend(a, b)
-    a
-  end
-
-  def on_array(a)
-    a
-  end
-
-  def on_assign(a, b)
-    a
-  end
-
-  def on_assign_error(a)
-    a
-  end
-
-  def on_assoc_new(a, b)
-    a
-  end
-
-  def on_assoclist_from_args(a)
-    a
-  end
-
-  def on_bare_assoc_hash(a)
-    a
-  end
-
-  def on_begin(a)
-    a
-  end
-
-  def on_binary(a, b, c)
-    a
-  end
-
-  def on_blockvar_add_block(a, b)
-    a
-  end
-
-  def on_blockvar_add_star(a, b)
-    a
-  end
-
-  def on_blockvar_new(a)
-    a
-  end
-
-  def on_bodystmt(a, b, c, d)
-    a
-  end
-
-  def on_brace_block(a, b)
-    a
-  end
-
-  def on_break(a)
-    a
-  end
-
-  def on_call(a, b, c)
-    a
-  end
-
-  def on_case(a, b)
-    a
-  end
-
-  def on_class(a, b, c)
-    a
-  end
-
-  def on_class_name_error(a)
-    a
-  end
-
-  def on_command(a, b)
-    a
-  end
-
-  def on_command_call(a, b, c, d)
-    a
-  end
-
-  def on_const_ref(a)
-    a
-  end
-
-  def on_constpath_field(a, b)
-    a
-  end
-
-  def on_constpath_ref(a, b)
-    a
-  end
-
-  def on_def(a, b, c)
-    a
-  end
-
-  def on_defined(a)
-    a
-  end
-
-  def on_defs(a, b, c, d, e)
-    a
-  end
-
-  def on_do_block(a, b)
-    a
-  end
-
-  def on_dot2(a, b)
-    a
-  end
-
-  def on_dot3(a, b)
-    a
-  end
-
-  def on_dyna_symbol(a)
-    a
-  end
-
-  def on_else(a)
-    a
-  end
-
-  def on_elsif(a, b, c)
-    a
-  end
-
-  def on_ensure(a)
-    a
-  end
-
-  def on_fcall(a)
-    a
-  end
-
-  def on_field(a, b, c)
-    a
-  end
-
-  def on_for(a, b, c)
-    a
-  end
-
-  def on_hash(a)
-    a
-  end
-
-  def on_if(a, b, c)
-    a
-  end
-
-  def on_if_mod(a, b)
-    a
-  end
-
-  def on_ifop(a, b, c)
-    a
-  end
-
-  def on_iter_block(a, b)
-    a
-  end
-
-  def on_lambda(a, b)
-    a
-  end
-
-  def on_massign(a, b)
-    a
-  end
-
-  def on_method_add_arg(a, b)
-    a
-  end
-
-  def on_mlhs_add(a, b)
-    a
-  end
-
-  def on_mlhs_add_star(a, b)
-    a
-  end
-
-  def on_mlhs_new
-    nil
-  end
-
-  def on_mlhs_paren(a)
-    a
-  end
-
-  def on_module(a, b)
-    a
-  end
-
-  def on_mrhs_add(a, b)
-    a
-  end
-
-  def on_mrhs_add_star(a, b)
-    a
-  end
-
-  def on_mrhs_new
-    nil
-  end
-
-  def on_mrhs_new_from_arglist(a)
-    a
-  end
-
-  def on_next(a)
-    a
-  end
-
-  def on_opassign(a, b, c)
-    a
-  end
-
-  def on_param_error(a)
-    a
-  end
-
-  def on_params(a, b, c, d)
-    a
-  end
-
-  def on_paren(a)
-    a
-  end
-
-  def on_parse_error(a)
-    a
-  end
-
-  def on_pragma(a, b)
-    a
-  end
-
-  def on_program(a)
-    a
-  end
-
-  def on_qwords_add(a, b)
-    a
-  end
-
-  def on_qwords_new
-    nil
-  end
-
-  def on_redo
-    nil
-  end
-
-  def on_regexp_literal(a, b)
-    a
-  end
-
-  def on_rescue(a, b, c, d)
-    a
-  end
-
-  def on_rescue_mod(a, b)
-    a
-  end
-
-  def on_restparam(a)
-    a
-  end
-
-  def on_retry
-    nil
-  end
-
-  def on_return(a)
-    a
-  end
-
-  def on_return0
-    nil
-  end
-
-  def on_sclass(a, b)
-    a
-  end
-
-  def on_space(a)
-    a
-  end
-
-  def on_stmts_add(a, b)
-    a
-  end
-
-  def on_stmts_new
-    nil
-  end
-
-  def on_string_add(a, b)
-    a
-  end
-
-  def on_string_concat(a, b)
-    a
-  end
-
-  def on_string_content
-    nil
-  end
-
-  def on_string_dvar(a)
-    a
-  end
-
-  def on_string_embexpr(a)
-    a
-  end
-
-  def on_string_literal(a)
-    a
-  end
-
-  def on_super(a)
-    a
-  end
-
-  def on_symbol(a)
-    a
-  end
-
-  def on_symbol_literal(a)
-    a
-  end
-
-  def on_topconst_field(a)
-    a
-  end
-
-  def on_topconst_ref(a)
-    a
-  end
-
-  def on_unary(a, b)
-    a
-  end
-
-  def on_undef(a)
-    a
-  end
-
-  def on_unless(a, b, c)
-    a
-  end
-
-  def on_unless_mod(a, b)
-    a
-  end
-
-  def on_until(a, b)
-    a
-  end
-
-  def on_until_mod(a, b)
-    a
-  end
-
-  def on_var_alias(a, b)
-    a
-  end
-
-  def on_var_field(a)
-    a
-  end
-
-  def on_var_ref(a)
-    a
-  end
-
-  def on_void_stmt
-    nil
-  end
-
-  def on_when(a, b, c)
-    a
-  end
-
-  def on_while(a, b)
-    a
-  end
-
-  def on_while_mod(a, b)
-    a
-  end
-
-  def on_word_add(a, b)
-    a
-  end
-
-  def on_word_new
-    nil
-  end
-
-  def on_words_add(a, b)
-    a
-  end
-
-  def on_words_new
-    nil
-  end
-
-  def on_xstring_add(a, b)
-    a
-  end
-
-  def on_xstring_literal(a)
-    a
-  end
-
-  def on_xstring_new
-    nil
-  end
-
-  def on_yield(a)
-    a
-  end
-
-  def on_yield0
-    nil
-  end
-
-  def on_zsuper
-    nil
-  end
-
-  #
-  # Lexer Events
-  #
-
-  def on_CHAR(token)
-    token
-  end
-
-  def on___end__(token)
-    token
-  end
-
-  def on_backref(token)
-    token
-  end
-
-  def on_backtick(token)
-    token
-  end
-
-  def on_comma(token)
-    token
-  end
-
-  def on_comment(token)
-    token
-  end
-
-  def on_const(token)
-    token
-  end
-
-  def on_cvar(token)
-    token
-  end
-
-  def on_embdoc(token)
-    token
-  end
-
-  def on_embdoc_beg(token)
-    token
-  end
-
-  def on_embdoc_end(token)
-    token
-  end
-
-  def on_embexpr_beg(token)
-    token
-  end
-
-  def on_embexpr_end(token)
-    token
-  end
-
-  def on_embvar(token)
-    token
-  end
-
-  def on_float(token)
-    token
-  end
-
-  def on_gvar(token)
-    token
-  end
-
-  def on_heredoc_beg(token)
-    token
-  end
-
-  def on_heredoc_end(token)
-    token
-  end
-
-  def on_ident(token)
-    token
-  end
-
-  def on_ignored_nl(token)
-    token
-  end
-
-  def on_int(token)
-    token
-  end
-
-  def on_ivar(token)
-    token
-  end
-
-  def on_kw(token)
-    token
-  end
-
-  def on_label(token)
-    token
-  end
-
-  def on_lbrace(token)
-    token
-  end
-
-  def on_lbracket(token)
-    token
-  end
-
-  def on_lparen(token)
-    token
-  end
-
-  def on_nl(token)
-    token
-  end
-
-  def on_op(token)
-    token
-  end
-
-  def on_period(token)
-    token
-  end
-
-  def on_qwords_beg(token)
-    token
-  end
-
-  def on_rbrace(token)
-    token
-  end
-
-  def on_rbracket(token)
-    token
-  end
-
-  def on_regexp_beg(token)
-    token
-  end
-
-  def on_regexp_end(token)
-    token
-  end
-
-  def on_rparen(token)
-    token
-  end
-
-  def on_semicolon(token)
-    token
-  end
-
-  def on_sp(token)
-    token
-  end
-
-  def on_symbeg(token)
-    token
-  end
-
-  def on_tlambda(token)
-    token
-  end
-
-  def on_tlambda_arg(token)
-    token
-  end
-
-  def on_tlambeg(token)
-    token
-  end
-
-  def on_tstring_beg(token)
-    token
-  end
-
-  def on_tstring_content(token)
-    token
-  end
-
-  def on_tstring_end(token)
-    token
-  end
-
-  def on_words_beg(token)
-    token
-  end
-
-  def on_words_sep(token)
-    token
-  end
-end

ext/ripper/lib/ripper/core.rb.in

@@ -1,7 +1,7 @@
 #
 # ripper/core.rb
 #
-# Copyright (C) 2003,2004 Minero Aoki
+# Copyright (C) 2003-2005 Minero Aoki
 #
 # This program is free software.
 # You can distribute and/or modify this program under the Ruby License.

ext/ripper/test/check-event-arity.rb

@@ -1,24 +0,0 @@
-def main
-  invalid = false
-  table = {}
-  ARGF.each do |line|
-    next if /\A\#\s*define\s+s?dispatch\d/ === line
-    next if /ripper_dispatch\d/ === line
-    line.scan(/dispatch(\d)\((\w+)/) do |num, ev|
-      num = num.to_i
-      if data = table[ev]
-        locations, arity = data
-        unless num == arity
-          invalid = true
-          puts "arity differ [#{ev}]: #{ARGF.lineno}->#{num}; #{locations.join(',')}->#{arity}"
-        end
-        locations.push ARGF.lineno
-      else
-        table[ev] = [[ARGF.lineno], num.to_i]
-      end
-    end
-  end
-  exit 1 if invalid
-end
-
-main

ext/ripper/test/check-event-coverage.sh

@@ -1,15 +0,0 @@
-# $Id$
-
-RUBY=${RUBY:-ruby}
-status=0
-
-$RUBY tools/list-parse-event-ids.rb parse.y | awk '{print "on__" $1}' > list_a
-$RUBY test/list-called-events.rb | sort -u > list_b
-diff -u list_a list_b | grep '^-on' | sed 's/^-on__//' > list_diff
-if [ -s list_diff ]
-then
-    cat list_diff
-    status=1
-fi
-rm -f list_a list_b list_diff
-exit $status

ext/ripper/test/check-scanner-event-coverage.rb

@@ -1,18 +0,0 @@
-def main
-  not_tested = eventids() - tested_ids()
-  unless not_tested.empty?
-    puts not_tested
-    exit 1
-  end
-  exit 0
-end
-
-def eventids
-  File.read('eventids2.c').scan(/on__(\w+)/).flatten.uniq
-end
-
-def tested_ids
-  File.read('test/test_scanner_events.rb').scan(/def test_(\S+)/).flatten.uniq
-end
-
-main

ext/ripper/test/list-called-events.rb

@@ -1,12 +0,0 @@
-require 'ripper.so'
-
-class R < Ripper
-  def method_missing(mid, *args)
-    puts mid
-    args[0]
-  end
-  undef :warn
-end
-
-fname = (ARGV[0] || 'test/src_rb')
-R.new(File.read(fname), fname, 1).parse

ext/ripper/test/src_rb

@@ -1,363 +0,0 @@
-# comment
-=begin
-  embedded document
-=end
-
-# literal
-1
-1000000000000000000000
-1.0
-1.234e5
-1..2
-1...3
-:symbol
-:"dynamic #{sym_embexpr} symbol"
-[1,2,3]
-{1 => 2}
-'string'
-"string"
-"before #{str_embexpr} after"
-"str #@ivar str"
-"str #$gvar str"
-"string" "concat"
-`/bin/true`
-{1, 2, 3, 4}
-/regexp/
-/regexp_with_opt/mioe
-/regexp #{regexp_embexpr} after/
-%q[string]
-%Q[str#{str_embexpr}ing]
-%r[regexp]
-%w( a b c )
-%W( a#{w_emb}b c d )
-<<HERE
-heredoc line 1
-heredoc line 2
-heredoc line 3
-HERE
-
-# special variables
-true
-false
-nil
-self
-
-# def
-def a
-end
-def b()
-end
-def c(a)
-end
-def d(a,*rest)
-end
-def e(a,&block)
-end
-def f(a,*rest,&block)
-end
-def g(*rest)
-end
-def h(&block)
-end
-def i(*rest,&block)
-end
-def j(CONST)
-end
-def k(@ivar)
-end
-def l($gvar)
-end
-def n(@@cvar)
-end
-
-# alias
-alias x b
-alias $rest $'     # error
-alias $nth $1      # error
-
-# undef
-undef warn
-
-# class, module
-class C
-end
-module M
-end
-class cname
-end
-class << Object.new
-  def self.a
-  end
-end
-
-# field
-$a = 1
-$' = 0   # error
-$1 = 0   # error
-@a = 2
-@@a = 3
-a = 4
-a += 1
-a -= 1
-a *= 1
-a /= 1
-a &&= 1
-a ||= 1
-m.a = 5
-m.a += 1
-m.a &&= 1
-m.a ||= 1
-a[1] = 2
-a[1] += 1
-a[1] &&= 1
-a[1] ||= 1
-C = 1
-C::C = 1
-::C = 1
-def m
-  C = 1      # dynamic const assignment
-  C::C = 1   # dynamic const assignment
-  ::C = 1    # dynamic const assignment
-end
-
-# ref
-lvar = $a
-lvar = @a
-lvar = @@a
-lvar = Object
-lvar = C
-lvar = C::C
-lvar = ::C
-lvar = a[1]
-
-# unary operator
-+1
--1
-not 1
-!1
-~str
-
-# binary operator
-1 + 1
-1 - 1
-1 * 1
-1 / 1
-1 ** 1
-1 ^ 1
-1 & 1
-1 | 1
-1 && 1
-1 || 1
-
-# mlhs, mrhs
-a, b, c = list
-a, = list
-a, * = list
-a, *b = list
-a, (b, c), d, *e = list
-mlhs = 1, 2
-mlhs = 1, 2, 3, *list
-mlhs = *list
-
-# method call
-m
-m()
-m(a)
-m(a,a)
-m(*a)
-m(&b)
-m(a,*a)
-m(a,&b)
-m(a,*a,&b)
-m(1=>2)
-m(1=>2,*a)
-m(1=>2,&b)
-m(1=>2,*a,&b)
-m ()
-m (a)
-m (a,a)
-m (*a)
-m (&b)
-m (a,*a)
-m (a,&b)
-m (a,*a,&b)
-m (1=>2)
-m (1=>2,*a)
-m (1=>2,&b)
-m (1=>2,*a,&b)
-m a
-m a,a
-m *a
-m &b
-m a,*a
-m a,&b
-m a,*a,&b
-m 1=>2
-m 1=>2,*a
-m 1=>2,&b
-m 1=>2,*a,&b
-obj.m
-obj.m()
-obj.m(a)
-obj.m(a,a)
-obj.m(*a)
-obj.m(&b)
-obj.m(a,*a)
-obj.m(a,&b)
-obj.m(a,*a,&b)
-obj.m(1=>2)
-obj.m(1=>2,*a)
-obj.m(1=>2,&b)
-obj.m(1=>2,*a,&b)
-obj.m ()
-obj.m (a)
-obj.m (a,a)
-obj.m (*a)
-obj.m (&b)
-obj.m (a,*a)
-obj.m (a,&b)
-obj.m (a,*a,&b)
-obj.m (1=>2)
-obj.m (1=>2)
-obj.m (1=>2,*a)
-obj.m (1=>2,&b)
-obj.m (1=>2,*a,&b)
-obj.m a
-obj.m a,a
-obj.m *a
-obj.m &b
-obj.m a,*a
-obj.m a,&b
-obj.m a,*a,&b
-obj.m 1=>2
-obj.m 1=>2,*a
-obj.m 1=>2,&b
-obj.m 1=>2,*a,&b
-
-# ambiguous argument
-m +1
-m /r/
-
-# iterator
-[1,2,3].each do |i|
-  print i
-end
-{1=>true}.each do |k,v|
-  puts k
-end
-[1,2,3].each {|i| print i }
-[1].each {|a,| }
-[1].each {|*b| }
-[1].each {|a,*b| }
-[1].each {|&block| }
-[1].each {|a,&block| }
-[1].each {|a,*b,&block| }
-a = lambda() {|n| n * n }
-a = lambda () {|n| n * n }
-a = lambda (a) {|n| n * n }
-a = lambda (a,b) {|n| n * n }
-
-# BEGIN, END
-BEGIN { }
-END { }
-
-# if, unless
-1 if true
-2 unless false
-if false
-  5
-elsif false
-  6
-elsif false then 7
-else
-  8
-end
-if m
-end
-unless 1
-  2
-end
-unless m
-end
-0 ? 1 : 2
-
-# case
-case 'a'
-when 'b'
-  ;
-when 'c' then 1
-else
-  2
-end
-case
-when 1
-when 2
-when 3
-else
-end
-case 1
-else
-end
-case
-else
-end
-
-# while, until, for
-while true
-  break
-  next
-  redo
-end
-begin
-  break
-end while true
-until false
-  break
-  next
-  redo
-end
-begin
-  break
-end until false
-for x in m()
-  break
-  next
-  redo
-end
-0 until true
-1 while false
-
-# begin, rescue, else, ensure
-begin
-  1
-rescue StandardError => er
-  2
-rescue => er
-  3
-  retry
-else
-  4
-ensure
-  5
-end
-a = 1 rescue 2
-
-# jumps
-def m
-  redo
-  yield
-  yield nil
-  super
-  super 1
-  return
-  return nil
-end
-
-# defined
-defined? f
-defined?(f)
-
-n = 1 \
-+ 1
-
-__END__

ext/ripper/test/validate.rb

@@ -1,68 +0,0 @@
-require 'ripper.so'
-
-class R < Ripper
-  def initialize(*args)
-    super
-    @lineno = 0
-  end
-
-  def parse
-    result = super
-    puts "#{@lineno}:result: #{rawVALUE(result)}"
-    validate_object result
-    p result
-    result
-  end
-
-  def on__nl(str)
-    @lineno += 1
-  end
-
-  def on__ignored_nl(str)
-    @lineno += 1
-  end
-
-  def on__comment(cmt)
-    @lineno += 1
-  end
-
-  def on__embdoc_beg(str)
-    @lineno += 1
-  end
-
-  def on__embdoc(str)
-    @lineno += 1
-  end
-
-  def on__embdoc_end(str)
-    @lineno += 1
-  end
-
-  def method_missing(mid, *args)
-    puts mid
-    args.each_with_index do |a,idx|
-      puts "#{@lineno}:#{mid}\##{idx+1}: #{rawVALUE(a)}"
-      validate_object a
-      p a
-    end
-    args[0]
-  end
-
-  def warn(*args)
-  end
-
-  def warning(*args)
-  end
-
-  unless respond_to?(:validate_object)
-    def validate_object(x)
-      x
-    end
-    def rawVALUE(x)
-      x.object_id
-    end
-  end
-end
-
-fname = (ARGV[0] || 'test/src_rb')
-R.new(File.read(fname), fname, 1).parse

ext/ripper/tools/generate-eventids1.rb

@@ -1,16 +0,0 @@
-# $Id$
-
-ids = File.readlines(ARGV[0]).map {|s| s.split[0] }
-
-ids.each do |id|
-  puts "static ID ripper_id_#{id};"
-end
-
-puts
-puts 'static void'
-puts 'ripper_init_eventids1()'
-puts '{'
-ids.each do |id|
-  puts %Q[    ripper_id_#{id} = rb_intern("on_#{id}");]
-end
-puts '}'

ext/ripper/tools/generate-ripper_rb.rb

@@ -1,60 +0,0 @@
-# $Id$
-
-def main
-  template, ids1, ids2 = *ARGV
-  print <<header
-# This file is automatically generated from #{File.basename(template)} and parse.y.
-# DO NOT MODIFY!!!!!!
-
-header
-  File.foreach(template) do |line|
-    case line
-    when /\A\#include ids1/
-      print_items read_ids(ids1)
-    when /\A\#include ids2/
-      print_items read_ids(ids2)
-    when /\A\#include handlers1/
-      File.foreach(ids1) do |line|
-        id, arity = line.split
-        arity = arity.to_i
-        puts
-        puts "  def on_#{id}#{paramdecl(arity)}"
-        puts "    #{arity == 0 ? 'nil' : 'a'}"
-        puts "  end"
-      end
-    when /\A\#include handlers2/
-      File.foreach(ids2) do |line|
-        id, arity = line.split
-        arity = arity.to_i
-        puts
-        puts "  def on_#{id}(token)"
-        puts "    token"
-        puts "  end"
-      end
-    when /\A\#include (.*)/
-      raise "unknown operation: #include #{$1}"
-    else
-      print line
-    end
-  end
-end
-
-def print_items(ids)
-  comma = ''
-  ids.each do |id, arity|
-    print comma; comma = ",\n"
-    print "    #{id.intern.inspect} => #{arity}"
-  end
-  puts
-end
-
-def read_ids(path)
-  File.readlines(path).map {|line| line.split }
-end
-
-def paramdecl(n)
-  return '' if n == 0
-  '(' + %w(a b c d e f g h i j k l m)[0, n].join(', ') + ')'
-end
-
-main

ext/ripper/tools/list-parse-event-ids.rb

@@ -1,39 +0,0 @@
-# $Id$
-
-def main
-  if ARGV[0] == '-a'
-    with_arity = true
-    ARGV.delete_at 0
-  else
-    with_arity = false
-  end
-  extract_ids(ARGF).each do |id, arity|
-    if with_arity
-      puts "#{id} #{arity}"
-    else
-      puts id
-    end
-  end
-end
-
-def extract_ids(f)
-  ids = {}
-  f.each do |line|
-    next if /\A\#\s*define\s+s?dispatch/ =~ line
-    next if /ripper_dispatch/ =~ line
-    if a = line.scan(/dispatch(\d)\((\w+)/)
-      a.each do |arity, event|
-        if ids[event]
-          unless ids[event] == arity.to_i
-            $stderr.puts "arity mismatch: #{event} (#{ids[event]} vs #{arity})"
-            exit 1
-          end
-        end
-        ids[event] = arity.to_i
-      end
-    end
-  end
-  ids.to_a.sort
-end
-
-main

ext/ripper/tools/list-scan-event-ids.rb

@@ -1,23 +0,0 @@
-# $Id$
-
-def main
-  if ARGV.first == '-a'
-    with_arity = true
-    ARGV.delete_at 0
-  else
-    with_arity = false
-  end
-  extract_ids(ARGF).sort.each do |id|
-    if with_arity
-      puts "#{id} 1"
-    else
-      puts id
-    end
-  end
-end
-
-def extract_ids(f)
-  f.read.scan(/ripper_id_(\w+)/).flatten.uniq
-end
-
-main

test/ripper/test_files.rb

@@ -13,6 +13,7 @@ class TestRipper_Generic < Test::Unit::TestCase
   def test_parse_files
     Find.find("#{SRCDIR}/lib", "#{SRCDIR}/ext", "#{SRCDIR}/sample", "#{SRCDIR}/test") {|n|
       next if /\.rb\z/ !~ n || !File.file?(n)
+$stderr.print '#'
       assert_nothing_raised { Parser.new(File.read(n)).parse }
     }
   end

test/ripper/test_parser_events.rb

@@ -1,12 +1,19 @@
-#
-# test_parser_events.rb
-#
-
 require 'dummyparser'
 require 'test/unit'
 
 class TestRipper_ParserEvents < Test::Unit::TestCase
 
+  # should be enabled
+=begin
+  def test_event_coverage
+    dispatched = Ripper::PARSER_EVENTS.map {|event,*| event }
+    dispatched.each do |e|
+      assert_equal true, respond_to?("test_#{e}", true),
+                   "event not tested: #{e.inspect}"
+    end
+  end
+=end
+
   def parse(str)
     DummyParser.new(str).parse.to_s
   end

test/ripper/test_scanner_events.rb

@@ -7,6 +7,13 @@ require 'test/unit'
 
 class TestRipper_ScannerEvents < Test::Unit::TestCase
 
+  def test_event_coverage
+    dispatched = Ripper::SCANNER_EVENTS.map {|event,_| event }
+    dispatched.each do |e|
+      assert_equal true, respond_to?("test_#{e}", true), "event not tested: #{e}"
+    end
+  end
+
   def scan(target, str)
     sym = "on_#{target}".intern
     Ripper.lex(str).select {|_,type,_| type == sym }.map {|_,_,tok| tok }
@@ -177,9 +184,9 @@ class TestRipper_ScannerEvents < Test::Unit::TestCase
                  scan('embexpr_beg', "m(<<EOS)\n\#{expr}\nEOS")
   end
 
-=begin
-  # currently detected as "rbrace"
   def test_embexpr_end
+=begin
+    # currently detected as "rbrace"
     assert_equal [],
                  scan('embexpr_end', '')
     assert_equal ['}'],
@@ -190,8 +197,8 @@ class TestRipper_ScannerEvents < Test::Unit::TestCase
                  scan('embexpr_end', '%Q[#{expr}]')
     assert_equal ['}'],
                  scan('embexpr_end', "m(<<EOS)\n\#{expr}\nEOS")
-  end
 =end
+  end
 
   def test_embvar
     assert_equal [],
@@ -781,4 +788,16 @@ class TestRipper_ScannerEvents < Test::Unit::TestCase
                  scan('CHAR', "@ivar")
   end
 
+  def test_label
+  end
+
+  def test_lambda
+  end
+
+  def test_lambeg
+  end
+
+  def test_lambda_arg
+  end
+
 end