ripper.y: fix word list events

* parse.y (parser_skip_words_sep): QWORDS_BEG should not include
  the first separators in ripper.

* parse.y (parser_parse_string): WORDS_SEP should not include
  the closing parentheses of a word list in ripper, should include
  spaces at beginning of lines.  [ruby-core:83864] [Bug #14126]

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@60883 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

parse.y

@@ -5819,6 +5819,26 @@ parser_str_new(const char *p, long n, rb_encoding *enc, int func, rb_encoding *e
 #define peekc() peekc_n(0)
 #define peekc_n(n) (lex_p+(n) < lex_pend ? (unsigned char)lex_p[n] : -1)
 
+#ifdef RIPPER
+static void
+parser_add_delayed_token(struct parser_params *parser, const char *tok, const char *end)
+{
+    if (tok < end) {
+	if (!has_delayed_token()) {
+	    parser->delayed = rb_str_buf_new(1024);
+	    rb_enc_associate(parser->delayed, current_enc);
+	    parser->delayed_line = ruby_sourceline;
+	    parser->delayed_col = (int)(tok - lex_pbeg);
+	}
+	rb_str_buf_cat(parser->delayed, tok, end - tok);
+	parser->tokp = end;
+    }
+}
+#define add_delayed_token(tok, end) parser_add_delayed_token(parser, (tok), (end))
+#else
+#define add_delayed_token(tok, end) ((void)(tok), (void)(end))
+#endif
+
 static int
 parser_nextline(struct parser_params *parser)
 {
@@ -5835,22 +5855,7 @@ parser_nextline(struct parser_params *parser)
 	}
 	parser->cr_seen = FALSE;
     }
-#ifdef RIPPER
+    add_delayed_token(parser->tokp, lex_pend);
-    if (parser->tokp < lex_pend) {
-	if (!has_delayed_token()) {
-	    parser->delayed = rb_str_buf_new(1024);
-	    rb_enc_associate(parser->delayed, current_enc);
-	    rb_str_buf_cat(parser->delayed,
-			   parser->tokp, lex_pend - parser->tokp);
-	    parser->delayed_line = ruby_sourceline;
-	    parser->delayed_col = (int)(parser->tokp - lex_pbeg);
-	}
-	else {
-	    rb_str_buf_cat(parser->delayed,
-			   parser->tokp, lex_pend - parser->tokp);
-	}
-    }
-#endif
     if (heredoc_end > 0) {
 	ruby_sourceline = heredoc_end;
 	heredoc_end = 0;
@@ -6597,6 +6602,9 @@ parser_parse_string(struct parser_params *parser, rb_strterm_literal_t *quote)
     VALUE lit;
 
     if (func & STR_FUNC_TERM) {
+#ifdef RIPPER
+	if (func & STR_FUNC_QWORDS) nextc(); /* delayed term */
+#endif
 	SET_LEX_STATE(EXPR_END|EXPR_ENDARG);
 	lex_strterm = 0;
 	return func & STR_FUNC_REGEXP ? tREGEXP_END : tSTRING_END;
@@ -6609,12 +6617,17 @@ parser_parse_string(struct parser_params *parser, rb_strterm_literal_t *quote)
     if (c == term && !quote->u0.nest) {
 	if (func & STR_FUNC_QWORDS) {
 	    quote->u1.func |= STR_FUNC_TERM;
+#ifdef RIPPER
+	    pushback(c); /* dispatch the term at tSTRING_END */
+#endif
+	    add_delayed_token(parser->tokp, lex_p);
 	    return ' ';
 	}
 	return parser_string_term(parser, func);
     }
     if (space) {
 	pushback(c);
+	add_delayed_token(parser->tokp, lex_p);
 	return ' ';
     }
     newtok();
@@ -7853,6 +7866,19 @@ parse_qmark(struct parser_params *parser, int space_seen)
     return tCHAR;
 }
 
+#ifndef RIPPER
+static void
+parser_skip_words_sep(struct parser_params *parser)
+{
+    int c;
+    do {c = nextc();} while (ISSPACE(c));
+    pushback(c);
+}
+#define skip_words_sep() parser_skip_words_sep(parser)
+#else
+#define skip_words_sep() ((void)0)
+#endif
+
 static enum yytokentype
 parse_percent(struct parser_params *parser, const int space_seen, const enum lex_state_e last_state)
 {
@@ -7897,26 +7923,22 @@ parse_percent(struct parser_params *parser, const int space_seen, const enum lex
 
 	  case 'W':
 	    lex_strterm = NEW_STRTERM(str_dword, term, paren);
-	    do {c = nextc();} while (ISSPACE(c));
+	    skip_words_sep();
-	    pushback(c);
 	    return tWORDS_BEG;
 
 	  case 'w':
 	    lex_strterm = NEW_STRTERM(str_sword, term, paren);
-	    do {c = nextc();} while (ISSPACE(c));
+	    skip_words_sep();
-	    pushback(c);
 	    return tQWORDS_BEG;
 
 	  case 'I':
 	    lex_strterm = NEW_STRTERM(str_dword, term, paren);
-	    do {c = nextc();} while (ISSPACE(c));
+	    skip_words_sep();
-	    pushback(c);
 	    return tSYMBOLS_BEG;
 
 	  case 'i':
 	    lex_strterm = NEW_STRTERM(str_sword, term, paren);
-	    do {c = nextc();} while (ISSPACE(c));
+	    skip_words_sep();
-	    pushback(c);
 	    return tQSYMBOLS_BEG;
 
 	  case 'x':

test/ripper/test_scanner_events.rb

@@ -637,8 +637,10 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase
                  scan('words_beg', '%W()')
     assert_equal ['%W('],
                  scan('words_beg', '%W(w w w)')
-    assert_equal ['%W( '],
+    assert_equal ['%W('],
                  scan('words_beg', '%W( w w w )')
+    assert_equal ['%W('],
+                 scan('words_beg', "%W(\nw)")
   end
 
   def test_qwords_beg
@@ -648,8 +650,10 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase
                  scan('qwords_beg', '%w()')
     assert_equal ['%w('],
                  scan('qwords_beg', '%w(w w w)')
-    assert_equal ['%w( '],
+    assert_equal ['%w('],
                  scan('qwords_beg', '%w( w w w )')
+    assert_equal ['%w('],
+                 scan('qwords_beg', "%w(\nw)")
   end
 
   def test_qsymbols_beg
@@ -659,8 +663,10 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase
                  scan('qsymbols_beg', '%i()')
     assert_equal ['%i('],
                  scan('qsymbols_beg', '%i(w w w)')
-    assert_equal ['%i( '],
+    assert_equal ['%i('],
                  scan('qsymbols_beg', '%i( w w w )')
+    assert_equal ['%i('],
+                 scan('qsymbols_beg', "%i(\nw)")
   end
 
   def test_symbols_beg
@@ -670,22 +676,25 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase
                  scan('symbols_beg', '%I()')
     assert_equal ['%I('],
                  scan('symbols_beg', '%I(w w w)')
-    assert_equal ['%I( '],
+    assert_equal ['%I('],
                  scan('symbols_beg', '%I( w w w )')
+    assert_equal ['%I('],
+                 scan('symbols_beg', "%I(\nw)")
   end
 
-  # FIXME: Close paren must not present (`words_end' scanner event?).
   def test_words_sep
     assert_equal [],
                  scan('words_sep', '')
-    assert_equal [')'],
+    assert_equal [],
                  scan('words_sep', '%w()')
-    assert_equal [' ', ' ', ')'],
+    assert_equal [' ', ' '],
                  scan('words_sep', '%w(w w w)')
-    assert_equal [' ', ' ', ' )'],
+    assert_equal [' ', ' ', ' ', ' '],
                  scan('words_sep', '%w( w w w )')
-    assert_equal ["\n", ' ', ' )'],
+    assert_equal [' ', "\n", ' ', ' '],
                  scan('words_sep', "%w( w\nw w )")
+    assert_equal ["\n\n", "\n ", ' ', ' '],
+                 scan('words_sep', "%w(\n\nw\n w w )")
   end
 
   def test_heredoc_beg