1berry/ruby
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ruby/yarp] Simplify creation of numerics

Browse Source 
https://github.com/ruby/yarp/commit/e5f6ffa23a
This commit is contained in:
Kevin Newton 2023-08-02 09:49:26 -04:00 committed by Takashi Kokubun
parent b6f26c2e4a
commit 3d032cf3e8
Notes: git 2023-08-17 00:48:10 +00:00 
Merged: https://github.com/ruby/ruby/pull/8226
4 changed files with 174 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
lib/yarp/lex_compat.rb
View File

@ -55,6 +55,9 @@ module YARP
EQUAL_GREATER: :on_op,
EQUAL_TILDE: :on_op,
FLOAT: :on_float,
FLOAT_IMAGINARY: :on_imaginary,
FLOAT_RATIONAL: :on_rational,
FLOAT_RATIONAL_IMAGINARY: :on_imaginary,
GREATER: :on_op,
GREATER_EQUAL: :on_op,
GREATER_GREATER: :on_op,
@ -64,8 +67,10 @@ module YARP
HEREDOC_START: :on_heredoc_beg,
IDENTIFIER: :on_ident,
IGNORED_NEWLINE: :on_ignored_nl,
IMAGINARY_NUMBER: :on_imaginary,
INTEGER: :on_int,
INTEGER_IMAGINARY: :on_imaginary,
INTEGER_RATIONAL: :on_rational,
INTEGER_RATIONAL_IMAGINARY: :on_imaginary,
INSTANCE_VARIABLE: :on_ivar,
INVALID: :INVALID,
KEYWORD___ENCODING__: :on_kw,
@ -145,7 +150,8 @@ module YARP
PLUS: :on_op,
PLUS_EQUAL: :on_op,
QUESTION_MARK: :on_op,
RATIONAL_NUMBER: :on_rational,
RATIONAL_FLOAT: :on_rational,
RATIONAL_INTEGER: :on_rational,
REGEXP_BEGIN: :on_regexp_beg,
REGEXP_END: :on_regexp_end,
SEMICOLON: :on_semicolon,

10
test/newline_test.rb
View File

@ -38,15 +38,15 @@ class NewlineTest < Test::Unit::TestCase
expected.delete_at actual.index(62)
elsif relative == "lib/yarp/lex_compat.rb"
# extra flag for: dedent_next =\n ((token.event: due to bytecode order
actual.delete(514)
actual.delete(520)
# different line for: token =\n case event: due to bytecode order
actual.delete(571)
expected.delete(572)
actual.delete(577)
expected.delete(578)
# extra flag for: lex_state =\n if RIPPER: due to bytecode order
actual.delete(604)
actual.delete(610)
# extra flag for: (token[2].start_with?("\#$") || token[2].start_with?("\#@"))
# unclear when ParenthesesNode should allow a second flag on the same line or not
actual.delete(731)
actual.delete(737)
end
assert_equal expected, actual

11
yarp/parser.h
View File

@ -96,10 +96,7 @@ typedef struct yp_lex_mode {
// This state is used when we are lexing a string or a string-like
// token, as in string content with either quote or an xstring.
YP_LEX_STRING,
// you lexed a number with extra information attached
YP_LEX_NUMERIC,
YP_LEX_STRING
} mode;
union {
@ -163,12 +160,6 @@ typedef struct yp_lex_mode {
char breakpoints[6];
} string;
struct {
yp_token_type_t type;
const char *start;
const char *end;
} numeric;
struct {
// These pointers point to the beginning and end of the heredoc
// identifier.

287
yarp/yarp.c
View File

@ -111,7 +111,6 @@ debug_lex_mode(yp_parser_t *parser) {
case YP_LEX_LIST: fprintf(stderr, "LIST (terminator=%c, interpolation=%d)", lex_mode->as.list.terminator, lex_mode->as.list.interpolation); break;
case YP_LEX_REGEXP: fprintf(stderr, "REGEXP (terminator=%c)", lex_mode->as.regexp.terminator); break;
case YP_LEX_STRING: fprintf(stderr, "STRING (terminator=%c, interpolation=%d)", lex_mode->as.string.terminator, lex_mode->as.string.interpolation); break;
case YP_LEX_NUMERIC: fprintf(stderr, "NUMERIC (token_type=%s)", yp_token_type_to_str(lex_mode->as.numeric.type)); break;
}
lex_mode = lex_mode->prev;
@ -1916,6 +1915,69 @@ yp_float_node_create(yp_parser_t *parser, const yp_token_t *token) {
return node;
}
// Allocate and initialize a new FloatNode node from a FLOAT_IMAGINARY token.
static yp_imaginary_node_t *
yp_float_node_imaginary_create(yp_parser_t *parser, const yp_token_t *token) {
assert(token->type == YP_TOKEN_FLOAT_IMAGINARY);
yp_imaginary_node_t *node = YP_ALLOC_NODE(parser, yp_imaginary_node_t);
*node = (yp_imaginary_node_t) {
{
.type = YP_NODE_IMAGINARY_NODE,
.location = YP_LOCATION_TOKEN_VALUE(token)
},
.numeric = (yp_node_t *) yp_float_node_create(parser, &((yp_token_t) {
.type = YP_TOKEN_FLOAT,
.start = token->start,
.end = token->end - 1
}))
};
return node;
}
// Allocate and initialize a new FloatNode node from a FLOAT_RATIONAL token.
static yp_rational_node_t *
yp_float_node_rational_create(yp_parser_t *parser, const yp_token_t *token) {
assert(token->type == YP_TOKEN_FLOAT_RATIONAL);
yp_rational_node_t *node = YP_ALLOC_NODE(parser, yp_rational_node_t);
*node = (yp_rational_node_t) {
{
.type = YP_NODE_RATIONAL_NODE,
.location = YP_LOCATION_TOKEN_VALUE(token)
},
.numeric = (yp_node_t *) yp_float_node_create(parser, &((yp_token_t) {
.type = YP_TOKEN_FLOAT,
.start = token->start,
.end = token->end - 1
}))
};
return node;
}
// Allocate and initialize a new FloatNode node from a FLOAT_RATIONAL_IMAGINARY token.
static yp_imaginary_node_t *
yp_float_node_rational_imaginary_create(yp_parser_t *parser, const yp_token_t *token) {
assert(token->type == YP_TOKEN_FLOAT_RATIONAL_IMAGINARY);
yp_imaginary_node_t *node = YP_ALLOC_NODE(parser, yp_imaginary_node_t);
*node = (yp_imaginary_node_t) {
{
.type = YP_NODE_IMAGINARY_NODE,
.location = YP_LOCATION_TOKEN_VALUE(token)
},
.numeric = (yp_node_t *) yp_float_node_rational_create(parser, &((yp_token_t) {
.type = YP_TOKEN_FLOAT_RATIONAL,
.start = token->start,
.end = token->end - 1
}))
};
return node;
}
// Allocate and initialize a new ForNode node.
static yp_for_node_t *
yp_for_node_create(
@ -2297,90 +2359,66 @@ yp_integer_node_create(yp_parser_t *parser, const yp_token_t *token) {
return node;
}
// Allocate and initialize a new RationalNode node.
static yp_rational_node_t *
yp_rational_node_create(yp_parser_t *parser, const yp_token_t *token) {
assert(token->type == YP_TOKEN_RATIONAL_NUMBER);
assert(parser->lex_modes.current->mode == YP_LEX_NUMERIC);
// Allocate and initialize a new IntegerNode node from an INTEGER_IMAGINARY token.
static yp_imaginary_node_t *
yp_integer_node_imaginary_create(yp_parser_t *parser, const yp_token_t *token) {
assert(token->type == YP_TOKEN_INTEGER_IMAGINARY);
yp_node_t *numeric_node;
yp_token_t numeric_token = {
.type = parser->lex_modes.current->as.numeric.type,
.start = parser->lex_modes.current->as.numeric.start,
.end = parser->lex_modes.current->as.numeric.end
yp_imaginary_node_t *node = YP_ALLOC_NODE(parser, yp_imaginary_node_t);
*node = (yp_imaginary_node_t) {
{
.type = YP_NODE_IMAGINARY_NODE,
.location = YP_LOCATION_TOKEN_VALUE(token)
},
.numeric = (yp_node_t *) yp_integer_node_create(parser, &((yp_token_t) {
.type = YP_TOKEN_INTEGER,
.start = token->start,
.end = token->end - 1
}))
};
switch (parser->lex_modes.current->as.numeric.type) {
case YP_TOKEN_INTEGER: {
lex_mode_pop(parser);
numeric_node = (yp_node_t *)yp_integer_node_create(parser, &numeric_token);
break;
}
case YP_TOKEN_FLOAT: {
lex_mode_pop(parser);
numeric_node = (yp_node_t *)yp_float_node_create(parser, &numeric_token);
break;
}
default: {
lex_mode_pop(parser);
numeric_node = (yp_node_t *)yp_missing_node_create(parser, numeric_token.start, numeric_token.end);
(void)numeric_node; // Suppress clang-analyzer-deadcode.DeadStores warning
assert(false && "unreachable");
}
}
yp_rational_node_t *node = YP_ALLOC_NODE(parser, yp_rational_node_t);
*node = (yp_rational_node_t) {
{ .type = YP_NODE_RATIONAL_NODE, .location = YP_LOCATION_TOKEN_VALUE(token) },
.numeric = numeric_node,
};
assert(parser->lex_modes.current->mode != YP_LEX_NUMERIC);
return node;
}
// Allocate and initialize a new ImaginaryNode node.
static yp_imaginary_node_t *
yp_imaginary_node_create(yp_parser_t *parser, const yp_token_t *token) {
assert(token->type == YP_TOKEN_IMAGINARY_NUMBER);
assert(parser->lex_modes.current->mode == YP_LEX_NUMERIC);
// Allocate and initialize a new IntegerNode node from an INTEGER_RATIONAL token.
static yp_rational_node_t *
yp_integer_node_rational_create(yp_parser_t *parser, const yp_token_t *token) {
assert(token->type == YP_TOKEN_INTEGER_RATIONAL);
yp_node_t *numeric_node;
yp_token_t numeric_token = {
.type = parser->lex_modes.current->as.numeric.type,
.start = parser->lex_modes.current->as.numeric.start,
.end = parser->lex_modes.current->as.numeric.end
yp_rational_node_t *node = YP_ALLOC_NODE(parser, yp_rational_node_t);
*node = (yp_rational_node_t) {
{
.type = YP_NODE_RATIONAL_NODE,
.location = YP_LOCATION_TOKEN_VALUE(token)
},
.numeric = (yp_node_t *) yp_integer_node_create(parser, &((yp_token_t) {
.type = YP_TOKEN_INTEGER,
.start = token->start,
.end = token->end - 1
}))
};
switch (parser->lex_modes.current->as.numeric.type) {
case YP_TOKEN_INTEGER: {
lex_mode_pop(parser);
numeric_node = (yp_node_t *)yp_integer_node_create(parser, &numeric_token);
break;
}
case YP_TOKEN_FLOAT: {
lex_mode_pop(parser);
numeric_node = (yp_node_t *)yp_float_node_create(parser, &numeric_token);
break;
}
case YP_TOKEN_RATIONAL_NUMBER: {
lex_mode_pop(parser);
numeric_node = (yp_node_t *)yp_rational_node_create(parser, &numeric_token);
break;
}
default: {
lex_mode_pop(parser);
numeric_node = (yp_node_t *)yp_missing_node_create(parser, numeric_token.start, numeric_token.end);
(void)numeric_node; // Suppress clang-analyzer-deadcode.DeadStores warning
assert(false && "unreachable");
}
}
return node;
}
// Allocate and initialize a new IntegerNode node from an INTEGER_RATIONAL_IMAGINARY token.
static yp_imaginary_node_t *
yp_integer_node_rational_imaginary_create(yp_parser_t *parser, const yp_token_t *token) {
assert(token->type == YP_TOKEN_INTEGER_RATIONAL_IMAGINARY);
yp_imaginary_node_t *node = YP_ALLOC_NODE(parser, yp_imaginary_node_t);
*node = (yp_imaginary_node_t) {
{ .type = YP_NODE_IMAGINARY_NODE, .location = YP_LOCATION_TOKEN_VALUE(token) },
.numeric = numeric_node
{
.type = YP_NODE_IMAGINARY_NODE,
.location = YP_LOCATION_TOKEN_VALUE(token)
},
.numeric = (yp_node_t *) yp_integer_node_rational_create(parser, &((yp_token_t) {
.type = YP_TOKEN_INTEGER_RATIONAL,
.start = token->start,
.end = token->end - 1
}))
};
assert(parser->lex_modes.current->mode != YP_LEX_NUMERIC);
return node;
}
@ -4775,37 +4813,6 @@ lex_numeric_prefix(yp_parser_t *parser) {
return type;
}
static yp_token_type_t
lex_finalize_numeric_type(yp_parser_t *parser, yp_token_type_t numeric_type, const char *numeric_end, const char *rational_end, const char *imaginary_end) {
if (rational_end || imaginary_end) {
lex_mode_push(parser, (yp_lex_mode_t) {
.mode = YP_LEX_NUMERIC,
.as.numeric.type = numeric_type,
.as.numeric.start = parser->current.start,
.as.numeric.end = numeric_end
});
}
if (rational_end && imaginary_end) {
lex_mode_push(parser, (yp_lex_mode_t) {
.mode = YP_LEX_NUMERIC,
.as.numeric.type = YP_TOKEN_RATIONAL_NUMBER,
.as.numeric.start = parser->current.start,
.as.numeric.end = rational_end
});
}
if (imaginary_end) {
return YP_TOKEN_IMAGINARY_NUMBER;
}
if (rational_end) {
return YP_TOKEN_RATIONAL_NUMBER;
}
return numeric_type;
}
static yp_token_type_t
lex_numeric(yp_parser_t *parser) {
yp_token_type_t type = YP_TOKEN_INTEGER;
@ -4814,22 +4821,35 @@ lex_numeric(yp_parser_t *parser) {
type = lex_numeric_prefix(parser);
const char *end = parser->current.end;
const char *rational_end = NULL;
const char *imaginary_end = NULL;
yp_token_type_t suffix_type = type;
if (match(parser, 'r')) {
rational_end = parser->current.end;
}
if (type == YP_TOKEN_INTEGER) {
if (match(parser, 'r')) {
suffix_type = YP_TOKEN_INTEGER_RATIONAL;
if (match(parser, 'i')) {
imaginary_end = parser->current.end;
if (match(parser, 'i')) {
suffix_type = YP_TOKEN_INTEGER_RATIONAL_IMAGINARY;
}
} else if (match(parser, 'i')) {
suffix_type = YP_TOKEN_INTEGER_IMAGINARY;
}
} else {
if (match(parser, 'r')) {
suffix_type = YP_TOKEN_FLOAT_RATIONAL;
if (match(parser, 'i')) {
suffix_type = YP_TOKEN_FLOAT_RATIONAL_IMAGINARY;
}
} else if (match(parser, 'i')) {
suffix_type = YP_TOKEN_FLOAT_IMAGINARY;
}
}
const unsigned char uc = (const unsigned char) peek(parser);
if (uc != '\0' && (uc >= 0x80 || ((uc >= 'a' && uc <= 'z') || (uc >= 'A' && uc <= 'Z')) || uc == '_')) {
parser->current.end = end;
} else {
type = lex_finalize_numeric_type(parser, type, end, rational_end, imaginary_end);
type = suffix_type;
}
}
@ -5449,7 +5469,6 @@ parser_lex(yp_parser_t *parser) {
case YP_LEX_DEFAULT:
case YP_LEX_EMBEXPR:
case YP_LEX_EMBVAR:
case YP_LEX_NUMERIC:
// We have a specific named label here because we are going to jump back to
// this location in the event that we have lexed a token that should not be
@ -8936,13 +8955,15 @@ parse_conditional(yp_parser_t *parser, yp_context_t context) {
// This macro allows you to define a case statement for all of the token types
// that represent the beginning of nodes that are "primitives" in a pattern
// matching expression.
#define YP_CASE_PRIMITIVE YP_TOKEN_INTEGER: case YP_TOKEN_FLOAT: case YP_TOKEN_RATIONAL_NUMBER: \
case YP_TOKEN_IMAGINARY_NUMBER: case YP_TOKEN_SYMBOL_BEGIN: case YP_TOKEN_REGEXP_BEGIN: case YP_TOKEN_BACKTICK: \
case YP_TOKEN_PERCENT_LOWER_X: case YP_TOKEN_PERCENT_LOWER_I: case YP_TOKEN_PERCENT_LOWER_W: \
case YP_TOKEN_PERCENT_UPPER_I: case YP_TOKEN_PERCENT_UPPER_W: case YP_TOKEN_STRING_BEGIN: case YP_TOKEN_KEYWORD_NIL: \
case YP_TOKEN_KEYWORD_SELF: case YP_TOKEN_KEYWORD_TRUE: case YP_TOKEN_KEYWORD_FALSE: case YP_TOKEN_KEYWORD___FILE__: \
case YP_TOKEN_KEYWORD___LINE__: case YP_TOKEN_KEYWORD___ENCODING__: case YP_TOKEN_MINUS_GREATER: \
case YP_TOKEN_HEREDOC_START: case YP_TOKEN_UMINUS_NUM: case YP_TOKEN_CHARACTER_LITERAL
#define YP_CASE_PRIMITIVE YP_TOKEN_INTEGER: case YP_TOKEN_INTEGER_IMAGINARY: case YP_TOKEN_INTEGER_RATIONAL: \
case YP_TOKEN_INTEGER_RATIONAL_IMAGINARY: case YP_TOKEN_FLOAT: case YP_TOKEN_FLOAT_IMAGINARY: \
case YP_TOKEN_FLOAT_RATIONAL: case YP_TOKEN_FLOAT_RATIONAL_IMAGINARY: case YP_TOKEN_SYMBOL_BEGIN: \
case YP_TOKEN_REGEXP_BEGIN: case YP_TOKEN_BACKTICK: case YP_TOKEN_PERCENT_LOWER_X: case YP_TOKEN_PERCENT_LOWER_I: \
case YP_TOKEN_PERCENT_LOWER_W: case YP_TOKEN_PERCENT_UPPER_I: case YP_TOKEN_PERCENT_UPPER_W: \
case YP_TOKEN_STRING_BEGIN: case YP_TOKEN_KEYWORD_NIL: case YP_TOKEN_KEYWORD_SELF: case YP_TOKEN_KEYWORD_TRUE: \
case YP_TOKEN_KEYWORD_FALSE: case YP_TOKEN_KEYWORD___FILE__: case YP_TOKEN_KEYWORD___LINE__: \
case YP_TOKEN_KEYWORD___ENCODING__: case YP_TOKEN_MINUS_GREATER: case YP_TOKEN_HEREDOC_START: \
case YP_TOKEN_UMINUS_NUM: case YP_TOKEN_CHARACTER_LITERAL
// This macro allows you to define a case statement for all of the token types
// that could begin a parameter.
@ -10305,7 +10326,16 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) {
}
case YP_TOKEN_FLOAT:
parser_lex(parser);
return (yp_node_t *)yp_float_node_create(parser, &parser->previous);
return (yp_node_t *) yp_float_node_create(parser, &parser->previous);
case YP_TOKEN_FLOAT_IMAGINARY:
parser_lex(parser);
return (yp_node_t *) yp_float_node_imaginary_create(parser, &parser->previous);
case YP_TOKEN_FLOAT_RATIONAL:
parser_lex(parser);
return (yp_node_t *) yp_float_node_rational_create(parser, &parser->previous);
case YP_TOKEN_FLOAT_RATIONAL_IMAGINARY:
parser_lex(parser);
return (yp_node_t *) yp_float_node_rational_imaginary_create(parser, &parser->previous);
case YP_TOKEN_NUMBERED_REFERENCE: {
parser_lex(parser);
yp_node_t *node = (yp_node_t *) yp_numbered_reference_read_node_create(parser, &parser->previous);
@ -10442,9 +10472,6 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) {
return node;
}
}
case YP_TOKEN_IMAGINARY_NUMBER:
parser_lex(parser);
return (yp_node_t *) yp_imaginary_node_create(parser, &parser->previous);
case YP_TOKEN_INSTANCE_VARIABLE: {
parser_lex(parser);
yp_node_t *node = (yp_node_t *) yp_instance_variable_read_node_create(parser, &parser->previous);
@ -10458,6 +10485,15 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) {
case YP_TOKEN_INTEGER:
parser_lex(parser);
return (yp_node_t *) yp_integer_node_create(parser, &parser->previous);
case YP_TOKEN_INTEGER_IMAGINARY:
parser_lex(parser);
return (yp_node_t *) yp_integer_node_imaginary_create(parser, &parser->previous);
case YP_TOKEN_INTEGER_RATIONAL:
parser_lex(parser);
return (yp_node_t *) yp_integer_node_rational_create(parser, &parser->previous);
case YP_TOKEN_INTEGER_RATIONAL_IMAGINARY:
parser_lex(parser);
return (yp_node_t *) yp_integer_node_rational_imaginary_create(parser, &parser->previous);
case YP_TOKEN_KEYWORD___ENCODING__:
parser_lex(parser);
return (yp_node_t *) yp_source_encoding_node_create(parser, &parser->previous);
@ -11617,9 +11653,6 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) {
return (yp_node_t *) array;
}
case YP_TOKEN_RATIONAL_NUMBER:
parser_lex(parser);
return (yp_node_t *) yp_rational_node_create(parser, &parser->previous);
case YP_TOKEN_REGEXP_BEGIN: {
yp_token_t opening = parser->current;
parser_lex(parser);