[ruby/prism] Add an IntegerField for parsing integer values

https://github.com/ruby/prism/commit/120d8c0479
This commit is contained in:
Kevin Newton 2024-02-22 10:55:29 -05:00
parent ff6ebba9de
commit af0a6ea1d5
17 changed files with 199 additions and 56 deletions

View File

@ -71,13 +71,6 @@ module Prism
end end
end end
class IntegerNode < Node
# Returns the value of the node as a Ruby Integer.
def value
Integer(slice)
end
end
class RationalNode < Node class RationalNode < Node
# Returns the value of the node as a Ruby Rational. # Returns the value of the node as a Ruby Rational.
def value def value

View File

@ -1867,6 +1867,9 @@ nodes:
- name: flags - name: flags
type: flags type: flags
kind: IntegerBaseFlags kind: IntegerBaseFlags
- name: value
type: integer
comment: The value of the integer literal as a number.
comment: | comment: |
Represents an integer number literal. Represents an integer number literal.

View File

@ -7,9 +7,9 @@
#include <ruby/encoding.h> #include <ruby/encoding.h>
#include "prism.h" #include "prism.h"
VALUE pm_source_new(pm_parser_t *parser, rb_encoding *encoding); VALUE pm_source_new(const pm_parser_t *parser, rb_encoding *encoding);
VALUE pm_token_new(pm_parser_t *parser, pm_token_t *token, rb_encoding *encoding, VALUE source); VALUE pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source);
VALUE pm_ast_new(pm_parser_t *parser, pm_node_t *node, rb_encoding *encoding, VALUE source); VALUE pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encoding, VALUE source);
void Init_prism_api_node(void); void Init_prism_api_node(void);
void Init_prism_pack(void); void Init_prism_pack(void);

View File

@ -3803,12 +3803,25 @@ pm_integer_node_create(pm_parser_t *parser, pm_node_flags_t base, const pm_token
assert(token->type == PM_TOKEN_INTEGER); assert(token->type == PM_TOKEN_INTEGER);
pm_integer_node_t *node = PM_ALLOC_NODE(parser, pm_integer_node_t); pm_integer_node_t *node = PM_ALLOC_NODE(parser, pm_integer_node_t);
*node = (pm_integer_node_t) {{ *node = (pm_integer_node_t) {
.type = PM_INTEGER_NODE, {
.flags = base | PM_NODE_FLAG_STATIC_LITERAL, .type = PM_INTEGER_NODE,
.location = PM_LOCATION_TOKEN_VALUE(token) .flags = base | PM_NODE_FLAG_STATIC_LITERAL,
}}; .location = PM_LOCATION_TOKEN_VALUE(token)
},
.value = { 0 }
};
pm_number_base_t number_base;
switch (base) {
case PM_INTEGER_BASE_FLAGS_BINARY: number_base = PM_NUMBER_BASE_BINARY; break;
case PM_INTEGER_BASE_FLAGS_OCTAL: number_base = PM_NUMBER_BASE_OCTAL; break;
case PM_INTEGER_BASE_FLAGS_DECIMAL: number_base = PM_NUMBER_BASE_DECIMAL; break;
case PM_INTEGER_BASE_FLAGS_HEXADECIMAL: number_base = PM_NUMBER_BASE_HEXADECIMAL; break;
default: assert(false && "unreachable");
}
pm_number_parse(&node->value, number_base, token->start, token->end);
return node; return node;
} }
@ -14281,6 +14294,9 @@ static inline void
parse_negative_numeric(pm_node_t *node) { parse_negative_numeric(pm_node_t *node) {
switch (PM_NODE_TYPE(node)) { switch (PM_NODE_TYPE(node)) {
case PM_INTEGER_NODE: case PM_INTEGER_NODE:
node->location.start--;
((pm_integer_node_t *) node)->value.negative = true;
break;
case PM_FLOAT_NODE: case PM_FLOAT_NODE:
node->location.start--; node->location.start--;
break; break;

View File

@ -12,13 +12,13 @@ static VALUE rb_cPrism<%= node.name %>;
<%- end -%> <%- end -%>
static VALUE static VALUE
pm_location_new(pm_parser_t *parser, const uint8_t *start, const uint8_t *end) { pm_location_new(const pm_parser_t *parser, const uint8_t *start, const uint8_t *end) {
uint64_t value = ((((uint64_t) (start - parser->start)) << 32) | ((uint32_t) (end - start))); uint64_t value = ((((uint64_t) (start - parser->start)) << 32) | ((uint32_t) (end - start)));
return ULL2NUM(value); return ULL2NUM(value);
} }
VALUE VALUE
pm_token_new(pm_parser_t *parser, pm_token_t *token, rb_encoding *encoding, VALUE source) { pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source) {
ID type = rb_intern(pm_token_type_name(token->type)); ID type = rb_intern(pm_token_type_name(token->type));
VALUE location = pm_location_new(parser, token->start, token->end); VALUE location = pm_location_new(parser, token->start, token->end);
@ -33,13 +33,30 @@ pm_token_new(pm_parser_t *parser, pm_token_t *token, rb_encoding *encoding, VALU
} }
static VALUE static VALUE
pm_string_new(pm_string_t *string, rb_encoding *encoding) { pm_string_new(const pm_string_t *string, rb_encoding *encoding) {
return rb_enc_str_new((const char *) pm_string_source(string), pm_string_length(string), encoding); return rb_enc_str_new((const char *) pm_string_source(string), pm_string_length(string), encoding);
} }
static VALUE
pm_integer_new(const pm_number_t *number) {
VALUE result = UINT2NUM(number->head.value);
size_t shift = 0;
for (const pm_number_node_t *node = number->head.next; node != NULL; node = node->next) {
VALUE receiver = rb_funcall(UINT2NUM(node->value), rb_intern("<<"), 1, ULONG2NUM(++shift * 32));
result = rb_funcall(receiver, rb_intern("|"), 1, result);
}
if (number->negative) {
result = rb_funcall(result, rb_intern("-@"), 0);
}
return result;
}
// Create a Prism::Source object from the given parser, after pm_parse() was called. // Create a Prism::Source object from the given parser, after pm_parse() was called.
VALUE VALUE
pm_source_new(pm_parser_t *parser, rb_encoding *encoding) { pm_source_new(const pm_parser_t *parser, rb_encoding *encoding) {
VALUE source_string = rb_enc_str_new((const char *) parser->start, parser->end - parser->start, encoding); VALUE source_string = rb_enc_str_new((const char *) parser->start, parser->end - parser->start, encoding);
VALUE offsets = rb_ary_new_capa(parser->newline_list.size); VALUE offsets = rb_ary_new_capa(parser->newline_list.size);
@ -53,12 +70,12 @@ pm_source_new(pm_parser_t *parser, rb_encoding *encoding) {
typedef struct pm_node_stack_node { typedef struct pm_node_stack_node {
struct pm_node_stack_node *prev; struct pm_node_stack_node *prev;
pm_node_t *visit; const pm_node_t *visit;
bool visited; bool visited;
} pm_node_stack_node_t; } pm_node_stack_node_t;
static void static void
pm_node_stack_push(pm_node_stack_node_t **stack, pm_node_t *visit) { pm_node_stack_push(pm_node_stack_node_t **stack, const pm_node_t *visit) {
pm_node_stack_node_t *node = malloc(sizeof(pm_node_stack_node_t)); pm_node_stack_node_t *node = malloc(sizeof(pm_node_stack_node_t));
node->prev = *stack; node->prev = *stack;
node->visit = visit; node->visit = visit;
@ -66,10 +83,10 @@ pm_node_stack_push(pm_node_stack_node_t **stack, pm_node_t *visit) {
*stack = node; *stack = node;
} }
static pm_node_t * static const pm_node_t *
pm_node_stack_pop(pm_node_stack_node_t **stack) { pm_node_stack_pop(pm_node_stack_node_t **stack) {
pm_node_stack_node_t *current = *stack; pm_node_stack_node_t *current = *stack;
pm_node_t *visit = current->visit; const pm_node_t *visit = current->visit;
*stack = current->prev; *stack = current->prev;
free(current); free(current);
@ -78,7 +95,7 @@ pm_node_stack_pop(pm_node_stack_node_t **stack) {
} }
VALUE VALUE
pm_ast_new(pm_parser_t *parser, pm_node_t *node, rb_encoding *encoding, VALUE source) { pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encoding, VALUE source) {
ID *constants = calloc(parser->constant_pool.size, sizeof(ID)); ID *constants = calloc(parser->constant_pool.size, sizeof(ID));
for (uint32_t index = 0; index < parser->constant_pool.size; index++) { for (uint32_t index = 0; index < parser->constant_pool.size; index++) {
@ -108,7 +125,7 @@ pm_ast_new(pm_parser_t *parser, pm_node_t *node, rb_encoding *encoding, VALUE so
continue; continue;
} }
pm_node_t *node = node_stack->visit; const pm_node_t *node = node_stack->visit;
node_stack->visited = true; node_stack->visited = true;
switch (PM_NODE_TYPE(node)) { switch (PM_NODE_TYPE(node)) {
@ -136,7 +153,7 @@ pm_ast_new(pm_parser_t *parser, pm_node_t *node, rb_encoding *encoding, VALUE so
} }
#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
} else { } else {
pm_node_t *node = pm_node_stack_pop(&node_stack); const pm_node_t *node = pm_node_stack_pop(&node_stack);
switch (PM_NODE_TYPE(node)) { switch (PM_NODE_TYPE(node)) {
<%- nodes.each do |node| -%> <%- nodes.each do |node| -%>
@ -193,6 +210,9 @@ pm_ast_new(pm_parser_t *parser, pm_node_t *node, rb_encoding *encoding, VALUE so
<%- when Prism::FlagsField -%> <%- when Prism::FlagsField -%>
#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
argv[<%= index %>] = ULONG2NUM(node->flags & ~PM_NODE_FLAG_COMMON_MASK); argv[<%= index %>] = ULONG2NUM(node->flags & ~PM_NODE_FLAG_COMMON_MASK);
<%- when Prism::IntegerField -%>
#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
argv[<%= index %>] = pm_integer_new(&cast-><%= field.name %>);
<%- else -%> <%- else -%>
<%- raise -%> <%- raise -%>
<%- end -%> <%- end -%>

View File

@ -183,6 +183,7 @@ typedef struct pm_<%= node.human %> {
when Prism::LocationField, Prism::OptionalLocationField then "pm_location_t #{field.name}" when Prism::LocationField, Prism::OptionalLocationField then "pm_location_t #{field.name}"
when Prism::UInt8Field then "uint8_t #{field.name}" when Prism::UInt8Field then "uint8_t #{field.name}"
when Prism::UInt32Field then "uint32_t #{field.name}" when Prism::UInt32Field then "uint32_t #{field.name}"
when Prism::IntegerField then "pm_number_t #{field.name}"
else raise field.class.name else raise field.class.name
end end
%>; %>;

View File

@ -133,7 +133,7 @@ module Prism
else else
table.field("<%= field.name %>", "[]") table.field("<%= field.name %>", "[]")
end end
<%- when Prism::StringField, Prism::ConstantField, Prism::OptionalConstantField, Prism::UInt8Field, Prism::UInt32Field, Prism::ConstantListField -%> <%- when Prism::StringField, Prism::ConstantField, Prism::OptionalConstantField, Prism::UInt8Field, Prism::UInt32Field, Prism::ConstantListField, Prism::IntegerField -%>
table.field("<%= field.name %>", node.<%= field.name %>.inspect) table.field("<%= field.name %>", node.<%= field.name %>.inspect)
<%- when Prism::LocationField -%> <%- when Prism::LocationField -%>
table.field("<%= field.name %>", location_inspect(node.<%= field.name %>)) table.field("<%= field.name %>", location_inspect(node.<%= field.name %>))

View File

@ -8,6 +8,7 @@ module Prism
# [ # [
# Prism::IntegerNode.new( # Prism::IntegerNode.new(
# Prism::IntegerBaseFlags::DECIMAL, # Prism::IntegerBaseFlags::DECIMAL,
# 1,
# Prism::Location.new(source, 1, 1), # Prism::Location.new(source, 1, 1),
# source # source
# ) # )
@ -22,7 +23,7 @@ module Prism
# source = Prism::Source.new("[1]") # source = Prism::Source.new("[1]")
# #
# ArrayNode( # ArrayNode(
# IntegerNode(Prism::IntegerBaseFlags::DECIMAL, Location(source, 1, 1)), source), # IntegerNode(Prism::IntegerBaseFlags::DECIMAL, 1, Location(source, 1, 1)), source),
# Location(source, 0, 1), # Location(source, 0, 1),
# Location(source, 2, 1), # Location(source, 2, 1),
# source # source

View File

@ -281,7 +281,7 @@ module Prism
inspector << "<%= pointer %><%= field.name %>:\n" inspector << "<%= pointer %><%= field.name %>:\n"
inspector << <%= field.name %>.inspect(inspector.child_inspector("<%= preadd %>")).delete_prefix(inspector.prefix) inspector << <%= field.name %>.inspect(inspector.child_inspector("<%= preadd %>")).delete_prefix(inspector.prefix)
end end
<%- when Prism::ConstantField, Prism::StringField, Prism::UInt8Field, Prism::UInt32Field -%> <%- when Prism::ConstantField, Prism::StringField, Prism::UInt8Field, Prism::UInt32Field, Prism::IntegerField -%>
inspector << "<%= pointer %><%= field.name %>: #{<%= field.name %>.inspect}\n" inspector << "<%= pointer %><%= field.name %>: #{<%= field.name %>.inspect}\n"
<%- when Prism::OptionalConstantField -%> <%- when Prism::OptionalConstantField -%>
if (<%= field.name %> = self.<%= field.name %>).nil? if (<%= field.name %> = self.<%= field.name %>).nil?

View File

@ -172,6 +172,17 @@ module Prism
(n >> 1) ^ (-(n & 1)) (n >> 1) ^ (-(n & 1))
end end
def load_integer
negative = io.getbyte != 0
length = load_varuint
value = 0
length.times { |index| value |= (load_varuint << (index * 32)) }
value = -value if negative
value
end
def load_serialized_length def load_serialized_length
io.read(4).unpack1("L") io.read(4).unpack1("L")
end end
@ -288,6 +299,7 @@ module Prism
when Prism::OptionalLocationField then "load_optional_location" when Prism::OptionalLocationField then "load_optional_location"
when Prism::UInt8Field then "io.getbyte" when Prism::UInt8Field then "io.getbyte"
when Prism::UInt32Field, Prism::FlagsField then "load_varuint" when Prism::UInt32Field, Prism::FlagsField then "load_varuint"
when Prism::IntegerField then "load_integer"
else raise else raise
end end
} + ["location"]).join(", ") -%>) } + ["location"]).join(", ") -%>)
@ -323,6 +335,7 @@ module Prism
when Prism::OptionalLocationField then "load_optional_location" when Prism::OptionalLocationField then "load_optional_location"
when Prism::UInt8Field then "io.getbyte" when Prism::UInt8Field then "io.getbyte"
when Prism::UInt32Field, Prism::FlagsField then "load_varuint" when Prism::UInt32Field, Prism::FlagsField then "load_varuint"
when Prism::IntegerField then "load_integer"
else raise else raise
end end
} + ["location"]).join(", ") -%>) } + ["location"]).join(", ") -%>)

View File

@ -74,6 +74,8 @@ pm_node_destroy(pm_parser_t *parser, pm_node_t *node) {
pm_node_list_free(parser, &cast-><%= field.name %>); pm_node_list_free(parser, &cast-><%= field.name %>);
<%- when Prism::ConstantListField -%> <%- when Prism::ConstantListField -%>
pm_constant_id_list_free(&cast-><%= field.name %>); pm_constant_id_list_free(&cast-><%= field.name %>);
<%- when Prism::IntegerField -%>
pm_number_free(&cast-><%= field.name %>);
<%- else -%> <%- else -%>
<%- raise -%> <%- raise -%>
<%- end -%> <%- end -%>
@ -103,14 +105,6 @@ pm_node_memsize_node(pm_node_t *node, pm_memsize_t *memsize) {
case <%= node.type %>: { case <%= node.type %>: {
pm_<%= node.human %>_t *cast = (pm_<%= node.human %>_t *) node; pm_<%= node.human %>_t *cast = (pm_<%= node.human %>_t *) node;
memsize->memsize += sizeof(*cast); memsize->memsize += sizeof(*cast);
<%- if node.fields.any? { |f| f.is_a?(Prism::NodeListField) } -%>
// Node lists will add in their own sizes below.
memsize->memsize -= sizeof(pm_node_list_t) * <%= node.fields.count { |f| f.is_a?(Prism::NodeListField) } %>;
<%- end -%>
<%- if node.fields.any? { |f| f.is_a?(Prism::ConstantListField) } -%>
// Constant id lists will add in their own sizes below.
memsize->memsize -= sizeof(pm_constant_id_list_t) * <%= node.fields.count { |f| f.is_a?(Prism::ConstantListField) } %>;
<%- end -%>
<%- node.fields.each do |field| -%> <%- node.fields.each do |field| -%>
<%- case field -%> <%- case field -%>
<%- when Prism::ConstantField, Prism::OptionalConstantField, Prism::UInt8Field, Prism::UInt32Field, Prism::FlagsField, Prism::LocationField, Prism::OptionalLocationField -%> <%- when Prism::ConstantField, Prism::OptionalConstantField, Prism::UInt8Field, Prism::UInt32Field, Prism::FlagsField, Prism::LocationField, Prism::OptionalLocationField -%>
@ -121,11 +115,13 @@ pm_node_memsize_node(pm_node_t *node, pm_memsize_t *memsize) {
pm_node_memsize_node((pm_node_t *)cast-><%= field.name %>, memsize); pm_node_memsize_node((pm_node_t *)cast-><%= field.name %>, memsize);
} }
<%- when Prism::StringField -%> <%- when Prism::StringField -%>
memsize->memsize += pm_string_memsize(&cast-><%= field.name %>); memsize->memsize += (pm_string_memsize(&cast-><%= field.name %>) - sizeof(pm_string_t));
<%- when Prism::NodeListField -%> <%- when Prism::NodeListField -%>
memsize->memsize += pm_node_list_memsize(&cast-><%= field.name %>, memsize); memsize->memsize += (pm_node_list_memsize(&cast-><%= field.name %>, memsize) - sizeof(pm_node_list_t));
<%- when Prism::ConstantListField -%> <%- when Prism::ConstantListField -%>
memsize->memsize += pm_constant_id_list_memsize(&cast-><%= field.name %>); memsize->memsize += (pm_constant_id_list_memsize(&cast-><%= field.name %>) - sizeof(pm_constant_id_list_t));
<%- when Prism::IntegerField -%>
memsize->memsize += (pm_number_memsize(&cast-><%= field.name %>) - sizeof(pm_number_t));
<%- else -%> <%- else -%>
<%- raise -%> <%- raise -%>
<%- end -%> <%- end -%>
@ -257,6 +253,29 @@ pm_dump_json(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *no
} }
<%- end -%> <%- end -%>
pm_buffer_append_byte(buffer, ']'); pm_buffer_append_byte(buffer, ']');
<%- when Prism::IntegerField -%>
{
const pm_number_t *number = &cast-><%= field.name %>;
if (number->length == 0) {
if (number->negative) pm_buffer_append_byte(buffer, '-');
pm_buffer_append_string(buffer, "%" PRIu32, number->head.value);
} else if (number->length == 1) {
if (number->negative) pm_buffer_append_byte(buffer, '-');
pm_buffer_append_format(buffer, "%" PRIu64, ((uint64_t) number->head.value) | (((uint64_t) number->head.next->value) << 32));
} else {
pm_buffer_append_byte(buffer, '{');
pm_buffer_append_format(buffer, "\"negative\": %s", number->negative ? "true" : "false");
pm_buffer_append_string(buffer, ",\"values\":[", 11);
const pm_number_node_t *node = &number->head;
while (node != NULL) {
pm_buffer_append_format(buffer, "%" PRIu32, node->value);
node = node->next;
if (node != NULL) pm_buffer_append_byte(buffer, ',');
}
pm_buffer_append_string(buffer, "]}", 2);
}
}
<%- else -%> <%- else -%>
<%- raise %> <%- raise %>
<%- end -%> <%- end -%>

View File

@ -126,6 +126,36 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm
<%- end -%> <%- end -%>
if (!found) pm_buffer_append_string(output_buffer, " nil", 4); if (!found) pm_buffer_append_string(output_buffer, " nil", 4);
pm_buffer_append_byte(output_buffer, '\n'); pm_buffer_append_byte(output_buffer, '\n');
<%- when Prism::IntegerField -%>
const pm_number_t *number = &cast-><%= field.name %>;
if (number->length == 0) {
pm_buffer_append_byte(output_buffer, ' ');
if (number->negative) pm_buffer_append_byte(output_buffer, '-');
pm_buffer_append_string(output_buffer, "%" PRIu32 "\n", number->head.value);
} else if (number->length == 1) {
pm_buffer_append_byte(output_buffer, ' ');
if (number->negative) pm_buffer_append_byte(output_buffer, '-');
pm_buffer_append_string(output_buffer, "%" PRIu64 "\n", ((uint64_t) number->head.value) | (((uint64_t) number->head.next->value) << 32));
} else {
pm_buffer_append_byte(output_buffer, ' ');
const pm_number_node_t *node = &number->head;
uint32_t index = 0;
while (node != NULL) {
if (index != 0) pm_buffer_append_string(output_buffer, " | ", 3);
pm_buffer_append_format(output_buffer, "%" PRIu32, node->value);
if (index != 0) {
pm_buffer_append_string(output_buffer, " << ", 4);
pm_buffer_append_format(output_buffer, "%" PRIu32, index * 32);
}
node = node->next;
index++;
}
pm_buffer_append_string(output_buffer, "]\n", 2);
}
<%- else -%> <%- else -%>
<%- raise -%> <%- raise -%>
<%- end -%> <%- end -%>

View File

@ -25,7 +25,7 @@ pm_serialize_location(const pm_parser_t *parser, const pm_location_t *location,
} }
static void static void
pm_serialize_string(pm_parser_t *parser, pm_string_t *string, pm_buffer_t *buffer) { pm_serialize_string(const pm_parser_t *parser, const pm_string_t *string, pm_buffer_t *buffer) {
switch (string->type) { switch (string->type) {
case PM_STRING_SHARED: { case PM_STRING_SHARED: {
pm_buffer_append_byte(buffer, 1); pm_buffer_append_byte(buffer, 1);
@ -49,6 +49,16 @@ pm_serialize_string(pm_parser_t *parser, pm_string_t *string, pm_buffer_t *buffe
} }
} }
static void
pm_serialize_integer(const pm_number_t *number, pm_buffer_t *buffer) {
pm_buffer_append_byte(buffer, number->negative ? 1 : 0);
pm_buffer_append_varuint(buffer, pm_sizet_to_u32(number->length + 1));
for (const pm_number_node_t *node = &number->head; node != NULL; node = node->next) {
pm_buffer_append_varuint(buffer, node->value);
}
}
static void static void
pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) { pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
pm_buffer_append_byte(buffer, (uint8_t) PM_NODE_TYPE(node)); pm_buffer_append_byte(buffer, (uint8_t) PM_NODE_TYPE(node));
@ -115,6 +125,8 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
pm_buffer_append_varuint(buffer, ((pm_<%= node.human %>_t *)node)-><%= field.name %>); pm_buffer_append_varuint(buffer, ((pm_<%= node.human %>_t *)node)-><%= field.name %>);
<%- when Prism::FlagsField -%> <%- when Prism::FlagsField -%>
pm_buffer_append_varuint(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK)); pm_buffer_append_varuint(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
<%- when Prism::IntegerField -%>
pm_serialize_integer(&((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
<%- else -%> <%- else -%>
<%- raise -%> <%- raise -%>
<%- end -%> <%- end -%>

View File

@ -254,6 +254,22 @@ module Prism
end end
end end
# This represents an arbitrarily-sized integer. When it gets to Ruby it will
# be an Integer.
class IntegerField < Field
def rbs_class
"Integer"
end
def rbi_class
"Integer"
end
def java_type
"VariableInteger"
end
end
# This class represents a node in the tree, configured by the config.yml file # This class represents a node in the tree, configured by the config.yml file
# in YAML format. It contains information about the name of the node and the # in YAML format. It contains information about the name of the node and the
# various child nodes it contains. # various child nodes it contains.
@ -315,6 +331,7 @@ module Prism
when "uint8" then UInt8Field when "uint8" then UInt8Field
when "uint32" then UInt32Field when "uint32" then UInt32Field
when "flags" then FlagsField when "flags" then FlagsField
when "integer" then IntegerField
else raise("Unknown field type: #{name.inspect}") else raise("Unknown field type: #{name.inspect}")
end end
end end

View File

@ -87,17 +87,11 @@ pm_number_parse_digit(const uint8_t character) {
*/ */
PRISM_EXPORTED_FUNCTION void PRISM_EXPORTED_FUNCTION void
pm_number_parse(pm_number_t *number, pm_number_base_t base, const uint8_t *start, const uint8_t *end) { pm_number_parse(pm_number_t *number, pm_number_base_t base, const uint8_t *start, const uint8_t *end) {
switch (*start) { // Ignore unary +. Unary + is parsed differently and will not end up here.
case '-': // Instead, it will modify the parsed number later.
number->negative = true; if (*start == '+') start++;
/* fallthrough */
case '+':
start++;
break;
default:
break;
}
// Determine the multiplier from the base, and skip past any prefixes.
uint32_t multiplier; uint32_t multiplier;
switch (base) { switch (base) {
case PM_NUMBER_BASE_BINARY: case PM_NUMBER_BASE_BINARY:
@ -133,13 +127,29 @@ pm_number_parse(pm_number_t *number, pm_number_base_t base, const uint8_t *start
break; break;
} }
for (pm_number_add(number, pm_number_parse_digit(*start++)); start < end; start++) { // It's possible that we've consumed everything at this point if there is an
// invalid number. If this is the case, we'll just return 0.
if (start >= end) return;
// Add the first digit to the number.
pm_number_add(number, pm_number_parse_digit(*start++));
// Add the subsequent digits to the number.
for (; start < end; start++) {
if (*start == '_') continue; if (*start == '_') continue;
pm_number_multiply(number, multiplier); pm_number_multiply(number, multiplier);
pm_number_add(number, pm_number_parse_digit(*start)); pm_number_add(number, pm_number_parse_digit(*start));
} }
} }
/**
* Return the memory size of the number.
*/
size_t
pm_number_memsize(const pm_number_t *number) {
return sizeof(pm_number_t) + number->length * sizeof(pm_number_node_t);
}
/** /**
* Recursively destroy the linked list of a number. * Recursively destroy the linked list of a number.
*/ */

View File

@ -84,6 +84,14 @@ typedef enum {
*/ */
PRISM_EXPORTED_FUNCTION void pm_number_parse(pm_number_t *number, pm_number_base_t base, const uint8_t *start, const uint8_t *end); PRISM_EXPORTED_FUNCTION void pm_number_parse(pm_number_t *number, pm_number_base_t base, const uint8_t *start, const uint8_t *end);
/**
* Return the memory size of the number.
*
* @param number The number to get the memory size of.
* @return The size of the memory associated with the number.
*/
size_t pm_number_memsize(const pm_number_t *number);
/** /**
* Free the internal memory of a number. This memory will only be allocated if * Free the internal memory of a number. This memory will only be allocated if
* the number exceeds the size of a single node in the linked list. * the number exceeds the size of a single node in the linked list.

View File

@ -966,8 +966,8 @@ module Prism
ParametersNode( ParametersNode(
[RequiredParameterNode(0, :a)], [RequiredParameterNode(0, :a)],
[ [
OptionalParameterNode(0, :b, Location(), Location(), IntegerNode(IntegerBaseFlags::DECIMAL)), OptionalParameterNode(0, :b, Location(), Location(), IntegerNode(IntegerBaseFlags::DECIMAL, 1)),
OptionalParameterNode(0, :d, Location(), Location(), IntegerNode(IntegerBaseFlags::DECIMAL)) OptionalParameterNode(0, :d, Location(), Location(), IntegerNode(IntegerBaseFlags::DECIMAL, 2))
], ],
nil, nil,
[RequiredParameterNode(0, :c), RequiredParameterNode(0, :e)], [RequiredParameterNode(0, :c), RequiredParameterNode(0, :e)],
@ -1024,7 +1024,7 @@ module Prism
Location(), Location(),
nil, nil,
nil, nil,
StatementsNode([IntegerNode(IntegerBaseFlags::DECIMAL)]), StatementsNode([IntegerNode(IntegerBaseFlags::DECIMAL, 42)]),
[], [],
Location(), Location(),
nil, nil,
@ -1214,7 +1214,7 @@ module Prism
:foo, :foo,
Location(), Location(),
nil, nil,
ParametersNode([], [OptionalParameterNode(0, :a, Location(), Location(), IntegerNode(IntegerBaseFlags::DECIMAL))], RestParameterNode(0, :c, Location(), Location()), [RequiredParameterNode(0, :b)], [], nil, nil), ParametersNode([], [OptionalParameterNode(0, :a, Location(), Location(), IntegerNode(IntegerBaseFlags::DECIMAL, 1))], RestParameterNode(0, :c, Location(), Location()), [RequiredParameterNode(0, :b)], [], nil, nil),
nil, nil,
[:a, :b, :c], [:a, :b, :c],
Location(), Location(),