From b9a2c96747cfac2bcc2883335b40f2a2d61d34cb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 21 Aug 2023 21:00:00 -0400 Subject: [PATCH] [ruby/yarp] Ensure interpolated symbols converted to regular symbols get opening and closing https://github.com/ruby/yarp/commit/386655d54f --- test/yarp/snapshots/patterns.txt | 27 +++++-- .../snapshots/seattlerb/dsym_esc_to_sym.txt | 4 +- test/yarp/snapshots/seattlerb/dsym_to_sym.txt | 4 +- .../unparser/corpus/literal/literal.txt | 17 +++-- .../snapshots/whitequark/interp_digit_var.txt | 4 +- ...ser_slash_slash_n_escaping_in_literals.txt | 2 +- yarp/config.yml | 2 +- yarp/yarp.c | 75 +++++++++---------- 8 files changed, 77 insertions(+), 58 deletions(-) diff --git a/test/yarp/snapshots/patterns.txt b/test/yarp/snapshots/patterns.txt index a55f1ba292..d5d37f8665 100644 --- a/test/yarp/snapshots/patterns.txt +++ b/test/yarp/snapshots/patterns.txt @@ -38,7 +38,7 @@ ProgramNode(0...3743)( ), MatchRequiredNode(78...91)( CallNode(78...81)(nil, nil, (78...81), nil, nil, nil, nil, 2, "foo"), - SymbolNode(85...91)(nil, (87...90), nil, "foo"), + SymbolNode(85...91)((85...87), (87...90), (90...91), "foo"), (82...84) ), MatchRequiredNode(92...104)( @@ -461,8 +461,8 @@ ProgramNode(0...3743)( "foo" ), RangeNode(472...488)( - SymbolNode(472...478)(nil, (474...477), nil, "foo"), - SymbolNode(482...488)(nil, (484...487), nil, "foo"), + SymbolNode(472...478)((472...474), (474...477), (477...478), "foo"), + SymbolNode(482...488)((482...484), (484...487), (487...488), "foo"), (479...481), 0 ), @@ -1943,7 +1943,12 @@ ProgramNode(0...3743)( 2, "foo" ), - SymbolNode(1733...1739)(nil, (1735...1738), nil, "foo"), + SymbolNode(1733...1739)( + (1733...1735), + (1735...1738), + (1738...1739), + "foo" + ), (1730...1732) ), MatchPredicateNode(1740...1752)( @@ -2442,7 +2447,12 @@ ProgramNode(0...3743)( "foo" ), [InNode(2196...2210)( - SymbolNode(2199...2205)(nil, (2201...2204), nil, "foo"), + SymbolNode(2199...2205)( + (2199...2201), + (2201...2204), + (2204...2205), + "foo" + ), nil, (2196...2198), (2206...2210) @@ -3134,7 +3144,12 @@ ProgramNode(0...3743)( (3000...3002), LocalVariableReadNode(3003...3006)(:baz, 0), StatementsNode(2993...2999)( - [SymbolNode(2993...2999)(nil, (2995...2998), nil, "foo")] + [SymbolNode(2993...2999)( + (2993...2995), + (2995...2998), + (2998...2999), + "foo" + )] ), nil, nil diff --git a/test/yarp/snapshots/seattlerb/dsym_esc_to_sym.txt b/test/yarp/snapshots/seattlerb/dsym_esc_to_sym.txt index 16cd806d01..4f6697237b 100644 --- a/test/yarp/snapshots/seattlerb/dsym_esc_to_sym.txt +++ b/test/yarp/snapshots/seattlerb/dsym_esc_to_sym.txt @@ -1,4 +1,6 @@ ProgramNode(0...17)( [], - StatementsNode(0...17)([SymbolNode(0...17)(nil, (2...16), nil, "Varietà")]) + StatementsNode(0...17)( + [SymbolNode(0...17)((0...2), (2...16), (16...17), "Varietà")] + ) ) diff --git a/test/yarp/snapshots/seattlerb/dsym_to_sym.txt b/test/yarp/snapshots/seattlerb/dsym_to_sym.txt index f2e3452dc1..d383b67184 100644 --- a/test/yarp/snapshots/seattlerb/dsym_to_sym.txt +++ b/test/yarp/snapshots/seattlerb/dsym_to_sym.txt @@ -2,8 +2,8 @@ ProgramNode(0...32)( [], StatementsNode(0...32)( [AliasNode(0...17)( - SymbolNode(6...11)(nil, (8...10), nil, "<<"), - SymbolNode(12...17)(nil, (14...16), nil, ">>"), + SymbolNode(6...11)((6...8), (8...10), (10...11), "<<"), + SymbolNode(12...17)((12...14), (14...16), (16...17), ">>"), (0...5) ), AliasNode(19...32)( diff --git a/test/yarp/snapshots/unparser/corpus/literal/literal.txt b/test/yarp/snapshots/unparser/corpus/literal/literal.txt index 9b0fad5072..bc9f86842d 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/literal.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/literal.txt @@ -276,11 +276,11 @@ ProgramNode(0...916)( XStringNode(435...439)((435...436), (436...438), (438...439), "`"), XStringNode(440...443)((440...441), (441...442), (442...443), "\""), SymbolNode(444...448)((444...445), (445...448), nil, "foo"), - SymbolNode(449...455)(nil, (451...454), nil, "A B"), + SymbolNode(449...455)((449...451), (451...454), (454...455), "A B"), SymbolNode(456...460)((456...457), (457...460), nil, "foo"), - SymbolNode(461...467)(nil, (463...466), nil, "A B"), - SymbolNode(468...475)(nil, (470...474), nil, "A\"B"), - InterpolatedSymbolNode(476...479)((476...478), [], (478...479)), + SymbolNode(461...467)((461...463), (463...466), (466...467), "A B"), + SymbolNode(468...475)((468...470), (470...474), (474...475), "A\"B"), + SymbolNode(476...479)((476...478), (0...0), (478...479), ""), RegularExpressionNode(480...485)( (480...481), (481...484), @@ -620,7 +620,7 @@ ProgramNode(0...916)( HashNode(828...843)( (828...829), [AssocNode(830...841)( - SymbolNode(830...836)(nil, (832...835), nil, "a b"), + SymbolNode(830...836)((830...832), (832...835), (835...836), "a b"), IntegerNode(840...841)(), (837...839) )], @@ -677,7 +677,12 @@ ProgramNode(0...916)( 0, "foo" ), - SymbolNode(893...901)(nil, (895...900), nil, "a\\\n" + "b"), + SymbolNode(893...901)( + (893...895), + (895...900), + (900...901), + "a\\\n" + "b" + ), InterpolatedXStringNode(902...916)( (902...903), [StringNode(903...907)(nil, (903...907), nil, " x\n"), diff --git a/test/yarp/snapshots/whitequark/interp_digit_var.txt b/test/yarp/snapshots/whitequark/interp_digit_var.txt index 4af55a97a8..48d7ac9671 100644 --- a/test/yarp/snapshots/whitequark/interp_digit_var.txt +++ b/test/yarp/snapshots/whitequark/interp_digit_var.txt @@ -83,8 +83,8 @@ ProgramNode(1...465)( "\#@@1", 0 ), - SymbolNode(294...300)(nil, (296...299), nil, "\#@1"), - SymbolNode(304...311)(nil, (306...310), nil, "\#@@1"), + SymbolNode(294...300)((294...296), (296...299), (299...300), "\#@1"), + SymbolNode(304...311)((304...306), (306...310), (310...311), "\#@@1"), SymbolNode(315...321)((315...317), (317...320), (320...321), "\#@1"), SymbolNode(325...332)((325...327), (327...331), (331...332), "\#@@1"), XStringNode(336...341)((336...337), (337...340), (340...341), "\#@1"), diff --git a/test/yarp/snapshots/whitequark/parser_slash_slash_n_escaping_in_literals.txt b/test/yarp/snapshots/whitequark/parser_slash_slash_n_escaping_in_literals.txt index 16db486beb..3a3377f96d 100644 --- a/test/yarp/snapshots/whitequark/parser_slash_slash_n_escaping_in_literals.txt +++ b/test/yarp/snapshots/whitequark/parser_slash_slash_n_escaping_in_literals.txt @@ -41,7 +41,7 @@ ProgramNode(0...210)( "ab", 0 ), - SymbolNode(123...130)(nil, (125...129), nil, "ab"), + SymbolNode(123...130)((123...125), (125...129), (129...130), "ab"), SymbolNode(132...139)((132...134), (134...138), (138...139), "ab"), InterpolatedStringNode(141...150)( (141...150), diff --git a/yarp/config.yml b/yarp/config.yml index 4cb060d80e..62a3c2249c 100644 --- a/yarp/config.yml +++ b/yarp/config.yml @@ -2084,7 +2084,7 @@ nodes: - name: opening_loc type: location? - name: value_loc - type: location + type: location? - name: closing_loc type: location? - name: unescaped diff --git a/yarp/yarp.c b/yarp/yarp.c index 5299ae9401..46a57fe4e2 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -2824,12 +2824,6 @@ yp_interpolated_symbol_node_append(yp_interpolated_symbol_node_t *node, yp_node_ node->base.location.end = part->location.end; } -static inline void -yp_interpolated_symbol_node_closing_set(yp_interpolated_symbol_node_t *node, const yp_token_t *closing) { - node->closing_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(closing); - node->base.location.end = closing->end; -} - // Allocate a new InterpolatedXStringNode node. static yp_interpolated_x_string_node_t * yp_interpolated_xstring_node_create(yp_parser_t *parser, const yp_token_t *opening, const yp_token_t *closing) { @@ -4057,20 +4051,20 @@ yp_symbol_node_label_p(yp_node_t *node) { // Convert the given StringNode node to a SymbolNode node. static yp_symbol_node_t * -yp_string_node_to_symbol_node(yp_parser_t *parser, yp_string_node_t *node) { +yp_string_node_to_symbol_node(yp_parser_t *parser, yp_string_node_t *node, const yp_token_t *opening, const yp_token_t *closing) { yp_symbol_node_t *new_node = YP_ALLOC_NODE(parser, yp_symbol_node_t); *new_node = (yp_symbol_node_t) { { .type = YP_NODE_SYMBOL_NODE, .location = { - .start = node->base.location.start - 2, - .end = node->base.location.end + 1 + .start = opening->start, + .end = closing->end } }, - .opening_loc = node->opening_loc, + .opening_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(opening), .value_loc = node->content_loc, - .closing_loc = node->closing_loc, + .closing_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(closing), .unescaped = node->unescaped }; @@ -9576,14 +9570,10 @@ parse_string_part(yp_parser_t *parser) { static yp_node_t * parse_symbol(yp_parser_t *parser, yp_lex_mode_t *lex_mode, yp_lex_state_t next_state) { - bool lex_string = lex_mode->mode == YP_LEX_STRING; - bool can_be_interpolated = lex_string && lex_mode->as.string.interpolation; yp_token_t opening = parser->previous; - if (!lex_string) { - if (next_state != YP_LEX_STATE_NONE) { - lex_state_set(parser, next_state); - } + if (lex_mode->mode != YP_LEX_STRING) { + if (next_state != YP_LEX_STATE_NONE) lex_state_set(parser, next_state); yp_token_t symbol; switch (parser->current.type) { @@ -9613,37 +9603,44 @@ parse_symbol(yp_parser_t *parser, yp_lex_mode_t *lex_mode, yp_lex_state_t next_s return (yp_node_t *) yp_symbol_node_create_and_unescape(parser, &opening, &symbol, &closing, YP_UNESCAPE_ALL); } - if (can_be_interpolated) { - // Create a node_list first. We'll use this to check if it should be an InterpolatedSymbolNode - // or a SymbolNode + if (lex_mode->as.string.interpolation) { + // If we have the end of the symbol, then we can return an empty symbol. + if (match_type_p(parser, YP_TOKEN_STRING_END)) { + if (next_state != YP_LEX_STATE_NONE) lex_state_set(parser, next_state); + parser_lex(parser); + + yp_token_t content = not_provided(parser); + yp_token_t closing = parser->previous; + return (yp_node_t *) yp_symbol_node_create_and_unescape(parser, &opening, &content, &closing, YP_UNESCAPE_NONE); + } + + // Now we can parse the first part of the symbol. + yp_node_t *part = parse_string_part(parser); + + // If we got a string part, then it's possible that we could transform + // what looks like an interpolated symbol into a regular symbol. + if (part && YP_NODE_TYPE_P(part, YP_NODE_STRING_NODE) && match_any_type_p(parser, 2, YP_TOKEN_STRING_END, YP_TOKEN_EOF)) { + if (next_state != YP_LEX_STATE_NONE) lex_state_set(parser, next_state); + parser_lex(parser); + + return (yp_node_t *) yp_string_node_to_symbol_node(parser, (yp_string_node_t *) part, &opening, &parser->previous); + } + + // Create a node_list first. We'll use this to check if it should be an + // InterpolatedSymbolNode or a SymbolNode. yp_node_list_t node_list = YP_EMPTY_NODE_LIST; + if (part) yp_node_list_append(&node_list, part); while (!match_any_type_p(parser, 2, YP_TOKEN_STRING_END, YP_TOKEN_EOF)) { - yp_node_t *part = parse_string_part(parser); - if (part != NULL) { + if ((part = parse_string_part(parser)) != NULL) { yp_node_list_append(&node_list, part); } } - yp_node_t *res; - // If the only element on the node_list is a StringNode, we know this is a SymbolNode - // and not an InterpolatedSymbolNode - if (node_list.size == 1 && YP_NODE_TYPE_P(node_list.nodes[0], YP_NODE_STRING_NODE)) { - res = (yp_node_t *)yp_string_node_to_symbol_node(parser, (yp_string_node_t *)node_list.nodes[0]); - free(node_list.nodes); - } - else { - yp_interpolated_symbol_node_t *interpolated = yp_interpolated_symbol_node_create(parser, &opening, &node_list, &opening); - yp_interpolated_symbol_node_closing_set(interpolated, &parser->current); - res = (yp_node_t *) interpolated; - } - - if (next_state != YP_LEX_STATE_NONE) { - lex_state_set(parser, next_state); - } + if (next_state != YP_LEX_STATE_NONE) lex_state_set(parser, next_state); expect(parser, YP_TOKEN_STRING_END, "Expected a closing delimiter for an interpolated symbol."); - return res; + return (yp_node_t *) yp_interpolated_symbol_node_create(parser, &opening, &node_list, &parser->previous); } yp_token_t content;