1berry/ruby
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ruby/prism] More consistent lex modes with %q

Browse Source 
https://github.com/ruby/prism/commit/014f714ed1
This commit is contained in:
Kevin Newton 2023-11-01 13:29:39 -04:00 committed by git
parent 336d81a671
commit db8803d583
1 changed files with 38 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
prism/prism.c
View File

@ -275,6 +275,16 @@ lex_mode_push_list(pm_parser_t *parser, bool interpolation, uint8_t delimiter) {
return lex_mode_push(parser, lex_mode); return lex_mode_push(parser, lex_mode);
} }
/**
* Push on a new list lex mode that is only used for compatibility. This is
* called when we're at the end of the file. We want the parser to be able to
* perform its normal error tolerance.
*/
static inline bool
lex_mode_push_list_eof(pm_parser_t *parser) {
return lex_mode_push_list(parser, false, '\0');
}
/** /**
* Push on a new regexp lex mode. * Push on a new regexp lex mode.
*/ */
@ -346,6 +356,16 @@ lex_mode_push_string(pm_parser_t *parser, bool interpolation, bool label_allowed
return lex_mode_push(parser, lex_mode); return lex_mode_push(parser, lex_mode);
} }
/**
* Push on a new string lex mode that is only used for compatibility. This is
* called when we're at the end of the file. We want the parser to be able to
* perform its normal error tolerance.
*/
static inline bool
lex_mode_push_string_eof(pm_parser_t *parser) {
return lex_mode_push_string(parser, false, false, '\0', '\0');
}
/** /**
* Pop the current lex state off the stack. If we're within the pre-allocated * Pop the current lex state off the stack. If we're within the pre-allocated
* space of the lex state stack, then we'll just decrement the index. Otherwise * space of the lex state stack, then we'll just decrement the index. Otherwise
@ -8791,6 +8811,8 @@ parser_lex(pm_parser_t *parser) {
if (parser->current.end < parser->end) { if (parser->current.end < parser->end) {
lex_mode_push_list(parser, false, *parser->current.end++); lex_mode_push_list(parser, false, *parser->current.end++);
} else {
lex_mode_push_list_eof(parser);
} }
LEX(PM_TOKEN_PERCENT_LOWER_I); LEX(PM_TOKEN_PERCENT_LOWER_I);
@ -8800,6 +8822,8 @@ parser_lex(pm_parser_t *parser) {
if (parser->current.end < parser->end) { if (parser->current.end < parser->end) {
lex_mode_push_list(parser, true, *parser->current.end++); lex_mode_push_list(parser, true, *parser->current.end++);
} else {
lex_mode_push_list_eof(parser);
} }
LEX(PM_TOKEN_PERCENT_UPPER_I); LEX(PM_TOKEN_PERCENT_UPPER_I);
@ -8811,6 +8835,8 @@ parser_lex(pm_parser_t *parser) {
lex_mode_push_regexp(parser, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end)); lex_mode_push_regexp(parser, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end));
pm_newline_list_check_append(&parser->newline_list, parser->current.end); pm_newline_list_check_append(&parser->newline_list, parser->current.end);
parser->current.end++; parser->current.end++;
} else {
lex_mode_push_regexp(parser, '\0', '\0');
} }
LEX(PM_TOKEN_REGEXP_BEGIN); LEX(PM_TOKEN_REGEXP_BEGIN);
@ -8822,6 +8848,8 @@ parser_lex(pm_parser_t *parser) {
lex_mode_push_string(parser, false, false, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end)); lex_mode_push_string(parser, false, false, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end));
pm_newline_list_check_append(&parser->newline_list, parser->current.end); pm_newline_list_check_append(&parser->newline_list, parser->current.end);
parser->current.end++; parser->current.end++;
} else {
lex_mode_push_string_eof(parser);
} }
LEX(PM_TOKEN_STRING_BEGIN); LEX(PM_TOKEN_STRING_BEGIN);
@ -8833,6 +8861,8 @@ parser_lex(pm_parser_t *parser) {
lex_mode_push_string(parser, true, false, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end)); lex_mode_push_string(parser, true, false, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end));
pm_newline_list_check_append(&parser->newline_list, parser->current.end); pm_newline_list_check_append(&parser->newline_list, parser->current.end);
parser->current.end++; parser->current.end++;
} else {
lex_mode_push_string_eof(parser);
} }
LEX(PM_TOKEN_STRING_BEGIN); LEX(PM_TOKEN_STRING_BEGIN);
@ -8844,6 +8874,8 @@ parser_lex(pm_parser_t *parser) {
lex_mode_push_string(parser, false, false, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end)); lex_mode_push_string(parser, false, false, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end));
lex_state_set(parser, PM_LEX_STATE_FNAME | PM_LEX_STATE_FITEM); lex_state_set(parser, PM_LEX_STATE_FNAME | PM_LEX_STATE_FITEM);
parser->current.end++; parser->current.end++;
} else {
lex_mode_push_string_eof(parser);
} }
LEX(PM_TOKEN_SYMBOL_BEGIN); LEX(PM_TOKEN_SYMBOL_BEGIN);
@ -8853,6 +8885,8 @@ parser_lex(pm_parser_t *parser) {
if (parser->current.end < parser->end) { if (parser->current.end < parser->end) {
lex_mode_push_list(parser, false, *parser->current.end++); lex_mode_push_list(parser, false, *parser->current.end++);
} else {
lex_mode_push_list_eof(parser);
} }
LEX(PM_TOKEN_PERCENT_LOWER_W); LEX(PM_TOKEN_PERCENT_LOWER_W);
@ -8862,6 +8896,8 @@ parser_lex(pm_parser_t *parser) {
if (parser->current.end < parser->end) { if (parser->current.end < parser->end) {
lex_mode_push_list(parser, true, *parser->current.end++); lex_mode_push_list(parser, true, *parser->current.end++);
} else {
lex_mode_push_list_eof(parser);
} }
LEX(PM_TOKEN_PERCENT_UPPER_W); LEX(PM_TOKEN_PERCENT_UPPER_W);
@ -8872,6 +8908,8 @@ parser_lex(pm_parser_t *parser) {
if (parser->current.end < parser->end) { if (parser->current.end < parser->end) {
lex_mode_push_string(parser, true, false, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end)); lex_mode_push_string(parser, true, false, lex_mode_incrementor(*parser->current.end), lex_mode_terminator(*parser->current.end));
parser->current.end++; parser->current.end++;
} else {
lex_mode_push_string_eof(parser);
} }
LEX(PM_TOKEN_PERCENT_LOWER_X); LEX(PM_TOKEN_PERCENT_LOWER_X);