ripper.y: fix word list events
* parse.y (parser_skip_words_sep): QWORDS_BEG should not include the first separators in ripper. * parse.y (parser_parse_string): WORDS_SEP should not include the closing parentheses of a word list in ripper, should include spaces at beginning of lines. [ruby-core:83864] [Bug #14126] git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@60883 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
nobu
parent
717b7fd9c4
commit
dee6a91002
70
parse.y
70
parse.y
@ -5819,6 +5819,26 @@ parser_str_new(const char *p, long n, rb_encoding *enc, int func, rb_encoding *e
|
||||
#define peekc() peekc_n(0)
|
||||
#define peekc_n(n) (lex_p+(n) < lex_pend ? (unsigned char)lex_p[n] : -1)
|
||||
|
||||
#ifdef RIPPER
|
||||
static void
|
||||
parser_add_delayed_token(struct parser_params *parser, const char *tok, const char *end)
|
||||
{
|
||||
if (tok < end) {
|
||||
if (!has_delayed_token()) {
|
||||
parser->delayed = rb_str_buf_new(1024);
|
||||
rb_enc_associate(parser->delayed, current_enc);
|
||||
parser->delayed_line = ruby_sourceline;
|
||||
parser->delayed_col = (int)(tok - lex_pbeg);
|
||||
}
|
||||
rb_str_buf_cat(parser->delayed, tok, end - tok);
|
||||
parser->tokp = end;
|
||||
}
|
||||
}
|
||||
#define add_delayed_token(tok, end) parser_add_delayed_token(parser, (tok), (end))
|
||||
#else
|
||||
#define add_delayed_token(tok, end) ((void)(tok), (void)(end))
|
||||
#endif
|
||||
|
||||
static int
|
||||
parser_nextline(struct parser_params *parser)
|
||||
{
|
||||
@ -5835,22 +5855,7 @@ parser_nextline(struct parser_params *parser)
|
||||
}
|
||||
parser->cr_seen = FALSE;
|
||||
}
|
||||
#ifdef RIPPER
|
||||
if (parser->tokp < lex_pend) {
|
||||
if (!has_delayed_token()) {
|
||||
parser->delayed = rb_str_buf_new(1024);
|
||||
rb_enc_associate(parser->delayed, current_enc);
|
||||
rb_str_buf_cat(parser->delayed,
|
||||
parser->tokp, lex_pend - parser->tokp);
|
||||
parser->delayed_line = ruby_sourceline;
|
||||
parser->delayed_col = (int)(parser->tokp - lex_pbeg);
|
||||
}
|
||||
else {
|
||||
rb_str_buf_cat(parser->delayed,
|
||||
parser->tokp, lex_pend - parser->tokp);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
add_delayed_token(parser->tokp, lex_pend);
|
||||
if (heredoc_end > 0) {
|
||||
ruby_sourceline = heredoc_end;
|
||||
heredoc_end = 0;
|
||||
@ -6597,6 +6602,9 @@ parser_parse_string(struct parser_params *parser, rb_strterm_literal_t *quote)
|
||||
VALUE lit;
|
||||
|
||||
if (func & STR_FUNC_TERM) {
|
||||
#ifdef RIPPER
|
||||
if (func & STR_FUNC_QWORDS) nextc(); /* delayed term */
|
||||
#endif
|
||||
SET_LEX_STATE(EXPR_END|EXPR_ENDARG);
|
||||
lex_strterm = 0;
|
||||
return func & STR_FUNC_REGEXP ? tREGEXP_END : tSTRING_END;
|
||||
@ -6609,12 +6617,17 @@ parser_parse_string(struct parser_params *parser, rb_strterm_literal_t *quote)
|
||||
if (c == term && !quote->u0.nest) {
|
||||
if (func & STR_FUNC_QWORDS) {
|
||||
quote->u1.func |= STR_FUNC_TERM;
|
||||
#ifdef RIPPER
|
||||
pushback(c); /* dispatch the term at tSTRING_END */
|
||||
#endif
|
||||
add_delayed_token(parser->tokp, lex_p);
|
||||
return ' ';
|
||||
}
|
||||
return parser_string_term(parser, func);
|
||||
}
|
||||
if (space) {
|
||||
pushback(c);
|
||||
add_delayed_token(parser->tokp, lex_p);
|
||||
return ' ';
|
||||
}
|
||||
newtok();
|
||||
@ -7853,6 +7866,19 @@ parse_qmark(struct parser_params *parser, int space_seen)
|
||||
return tCHAR;
|
||||
}
|
||||
|
||||
#ifndef RIPPER
|
||||
static void
|
||||
parser_skip_words_sep(struct parser_params *parser)
|
||||
{
|
||||
int c;
|
||||
do {c = nextc();} while (ISSPACE(c));
|
||||
pushback(c);
|
||||
}
|
||||
#define skip_words_sep() parser_skip_words_sep(parser)
|
||||
#else
|
||||
#define skip_words_sep() ((void)0)
|
||||
#endif
|
||||
|
||||
static enum yytokentype
|
||||
parse_percent(struct parser_params *parser, const int space_seen, const enum lex_state_e last_state)
|
||||
{
|
||||
@ -7897,26 +7923,22 @@ parse_percent(struct parser_params *parser, const int space_seen, const enum lex
|
||||
|
||||
case 'W':
|
||||
lex_strterm = NEW_STRTERM(str_dword, term, paren);
|
||||
do {c = nextc();} while (ISSPACE(c));
|
||||
pushback(c);
|
||||
skip_words_sep();
|
||||
return tWORDS_BEG;
|
||||
|
||||
case 'w':
|
||||
lex_strterm = NEW_STRTERM(str_sword, term, paren);
|
||||
do {c = nextc();} while (ISSPACE(c));
|
||||
pushback(c);
|
||||
skip_words_sep();
|
||||
return tQWORDS_BEG;
|
||||
|
||||
case 'I':
|
||||
lex_strterm = NEW_STRTERM(str_dword, term, paren);
|
||||
do {c = nextc();} while (ISSPACE(c));
|
||||
pushback(c);
|
||||
skip_words_sep();
|
||||
return tSYMBOLS_BEG;
|
||||
|
||||
case 'i':
|
||||
lex_strterm = NEW_STRTERM(str_sword, term, paren);
|
||||
do {c = nextc();} while (ISSPACE(c));
|
||||
pushback(c);
|
||||
skip_words_sep();
|
||||
return tQSYMBOLS_BEG;
|
||||
|
||||
case 'x':
|
||||
|
||||
@ -637,8 +637,10 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase
|
||||
scan('words_beg', '%W()')
|
||||
assert_equal ['%W('],
|
||||
scan('words_beg', '%W(w w w)')
|
||||
assert_equal ['%W( '],
|
||||
assert_equal ['%W('],
|
||||
scan('words_beg', '%W( w w w )')
|
||||
assert_equal ['%W('],
|
||||
scan('words_beg', "%W(\nw)")
|
||||
end
|
||||
|
||||
def test_qwords_beg
|
||||
@ -648,8 +650,10 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase
|
||||
scan('qwords_beg', '%w()')
|
||||
assert_equal ['%w('],
|
||||
scan('qwords_beg', '%w(w w w)')
|
||||
assert_equal ['%w( '],
|
||||
assert_equal ['%w('],
|
||||
scan('qwords_beg', '%w( w w w )')
|
||||
assert_equal ['%w('],
|
||||
scan('qwords_beg', "%w(\nw)")
|
||||
end
|
||||
|
||||
def test_qsymbols_beg
|
||||
@ -659,8 +663,10 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase
|
||||
scan('qsymbols_beg', '%i()')
|
||||
assert_equal ['%i('],
|
||||
scan('qsymbols_beg', '%i(w w w)')
|
||||
assert_equal ['%i( '],
|
||||
assert_equal ['%i('],
|
||||
scan('qsymbols_beg', '%i( w w w )')
|
||||
assert_equal ['%i('],
|
||||
scan('qsymbols_beg', "%i(\nw)")
|
||||
end
|
||||
|
||||
def test_symbols_beg
|
||||
@ -670,22 +676,25 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase
|
||||
scan('symbols_beg', '%I()')
|
||||
assert_equal ['%I('],
|
||||
scan('symbols_beg', '%I(w w w)')
|
||||
assert_equal ['%I( '],
|
||||
assert_equal ['%I('],
|
||||
scan('symbols_beg', '%I( w w w )')
|
||||
assert_equal ['%I('],
|
||||
scan('symbols_beg', "%I(\nw)")
|
||||
end
|
||||
|
||||
# FIXME: Close paren must not present (`words_end' scanner event?).
|
||||
def test_words_sep
|
||||
assert_equal [],
|
||||
scan('words_sep', '')
|
||||
assert_equal [')'],
|
||||
assert_equal [],
|
||||
scan('words_sep', '%w()')
|
||||
assert_equal [' ', ' ', ')'],
|
||||
assert_equal [' ', ' '],
|
||||
scan('words_sep', '%w(w w w)')
|
||||
assert_equal [' ', ' ', ' )'],
|
||||
assert_equal [' ', ' ', ' ', ' '],
|
||||
scan('words_sep', '%w( w w w )')
|
||||
assert_equal ["\n", ' ', ' )'],
|
||||
assert_equal [' ', "\n", ' ', ' '],
|
||||
scan('words_sep', "%w( w\nw w )")
|
||||
assert_equal ["\n\n", "\n ", ' ', ' '],
|
||||
scan('words_sep', "%w(\n\nw\n w w )")
|
||||
end
|
||||
|
||||
def test_heredoc_beg
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user