[ruby/prism] Get general prettyprint structure working

https://github.com/ruby/prism/commit/7c9cf63d4c
This commit is contained in:
Kevin Newton 2023-10-24 10:04:48 -04:00
parent 9792be3610
commit e9aa2398b9
6 changed files with 249 additions and 158 deletions

View File

@ -554,6 +554,27 @@ parse_serialize_file_metadata(VALUE self, VALUE filepath, VALUE metadata) {
return result;
}
static VALUE
rb_prism_debug_inspect_node(VALUE self, VALUE source) {
pm_string_t input;
input_load_string(&input, source);
pm_parser_t parser;
pm_parser_init(&parser, pm_string_source(&input), pm_string_length(&input), NULL);
pm_node_t *node = pm_parse(&parser);
pm_buffer_t buffer = { 0 };
pm_prettyprint(&parser, node, &buffer);
VALUE string = rb_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer));
pm_buffer_free(&buffer);
pm_node_destroy(&parser, node);
pm_parser_free(&parser);
return string;
}
/******************************************************************************/
/* Initialization of the extension */
/******************************************************************************/
@ -607,6 +628,8 @@ Init_prism(void) {
rb_define_singleton_method(rb_cPrismDebug, "profile_file", profile_file, 1);
rb_define_singleton_method(rb_cPrismDebug, "parse_serialize_file_metadata", parse_serialize_file_metadata, 2);
rb_define_singleton_method(rb_cPrismDebug, "inspect_node", rb_prism_debug_inspect_node, 1);
// Next, initialize the other APIs.
Init_prism_api_node();
Init_prism_pack();

View File

@ -6343,24 +6343,24 @@ escape_byte(uint8_t value, const uint8_t flags) {
static inline void
escape_write_unicode(pm_parser_t *parser, pm_buffer_t *buffer, const uint8_t *start, const uint8_t *end, uint32_t value) {
if (value <= 0x7F) { // 0xxxxxxx
pm_buffer_append_u8(buffer, (uint8_t) value);
pm_buffer_append_byte(buffer, (uint8_t) value);
} else if (value <= 0x7FF) { // 110xxxxx 10xxxxxx
pm_buffer_append_u8(buffer, (uint8_t) (0xC0 | (value >> 6)));
pm_buffer_append_u8(buffer, (uint8_t) (0x80 | (value & 0x3F)));
pm_buffer_append_byte(buffer, (uint8_t) (0xC0 | (value >> 6)));
pm_buffer_append_byte(buffer, (uint8_t) (0x80 | (value & 0x3F)));
} else if (value <= 0xFFFF) { // 1110xxxx 10xxxxxx 10xxxxxx
pm_buffer_append_u8(buffer, (uint8_t) (0xE0 | (value >> 12)));
pm_buffer_append_u8(buffer, (uint8_t) (0x80 | ((value >> 6) & 0x3F)));
pm_buffer_append_u8(buffer, (uint8_t) (0x80 | (value & 0x3F)));
pm_buffer_append_byte(buffer, (uint8_t) (0xE0 | (value >> 12)));
pm_buffer_append_byte(buffer, (uint8_t) (0x80 | ((value >> 6) & 0x3F)));
pm_buffer_append_byte(buffer, (uint8_t) (0x80 | (value & 0x3F)));
} else if (value <= 0x10FFFF) { // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
pm_buffer_append_u8(buffer, (uint8_t) (0xF0 | (value >> 18)));
pm_buffer_append_u8(buffer, (uint8_t) (0x80 | ((value >> 12) & 0x3F)));
pm_buffer_append_u8(buffer, (uint8_t) (0x80 | ((value >> 6) & 0x3F)));
pm_buffer_append_u8(buffer, (uint8_t) (0x80 | (value & 0x3F)));
pm_buffer_append_byte(buffer, (uint8_t) (0xF0 | (value >> 18)));
pm_buffer_append_byte(buffer, (uint8_t) (0x80 | ((value >> 12) & 0x3F)));
pm_buffer_append_byte(buffer, (uint8_t) (0x80 | ((value >> 6) & 0x3F)));
pm_buffer_append_byte(buffer, (uint8_t) (0x80 | (value & 0x3F)));
} else {
pm_parser_err(parser, start, end, PM_ERR_ESCAPE_INVALID_UNICODE);
pm_buffer_append_u8(buffer, 0xEF);
pm_buffer_append_u8(buffer, 0xBF);
pm_buffer_append_u8(buffer, 0xBD);
pm_buffer_append_byte(buffer, 0xEF);
pm_buffer_append_byte(buffer, 0xBF);
pm_buffer_append_byte(buffer, 0xBD);
}
}
@ -6386,18 +6386,18 @@ escape_write_byte(pm_buffer_t *buffer, uint8_t flags, uint8_t byte) {
uint8_t byte2 = (uint8_t) (byte & 0xF);
if (byte1 >= 0xA) {
pm_buffer_append_u8(buffer, (uint8_t) ((byte1 - 0xA) + 'A'));
pm_buffer_append_byte(buffer, (uint8_t) ((byte1 - 0xA) + 'A'));
} else {
pm_buffer_append_u8(buffer, (uint8_t) (byte1 + '0'));
pm_buffer_append_byte(buffer, (uint8_t) (byte1 + '0'));
}
if (byte2 >= 0xA) {
pm_buffer_append_u8(buffer, (uint8_t) (byte2 - 0xA + 'A'));
pm_buffer_append_byte(buffer, (uint8_t) (byte2 - 0xA + 'A'));
} else {
pm_buffer_append_u8(buffer, (uint8_t) (byte2 + '0'));
pm_buffer_append_byte(buffer, (uint8_t) (byte2 + '0'));
}
} else {
pm_buffer_append_u8(buffer, byte);
pm_buffer_append_byte(buffer, byte);
}
}
@ -6407,57 +6407,57 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, uint8_t flags) {
switch (peek(parser)) {
case '\\': {
parser->current.end++;
pm_buffer_append_u8(buffer, '\\');
pm_buffer_append_byte(buffer, '\\');
return;
}
case '\'': {
parser->current.end++;
pm_buffer_append_u8(buffer, '\'');
pm_buffer_append_byte(buffer, '\'');
return;
}
case 'a': {
parser->current.end++;
pm_buffer_append_u8(buffer, '\a');
pm_buffer_append_byte(buffer, '\a');
return;
}
case 'b': {
parser->current.end++;
pm_buffer_append_u8(buffer, '\b');
pm_buffer_append_byte(buffer, '\b');
return;
}
case 'e': {
parser->current.end++;
pm_buffer_append_u8(buffer, '\033');
pm_buffer_append_byte(buffer, '\033');
return;
}
case 'f': {
parser->current.end++;
pm_buffer_append_u8(buffer, '\f');
pm_buffer_append_byte(buffer, '\f');
return;
}
case 'n': {
parser->current.end++;
pm_buffer_append_u8(buffer, '\n');
pm_buffer_append_byte(buffer, '\n');
return;
}
case 'r': {
parser->current.end++;
pm_buffer_append_u8(buffer, '\r');
pm_buffer_append_byte(buffer, '\r');
return;
}
case 's': {
parser->current.end++;
pm_buffer_append_u8(buffer, ' ');
pm_buffer_append_byte(buffer, ' ');
return;
}
case 't': {
parser->current.end++;
pm_buffer_append_u8(buffer, '\t');
pm_buffer_append_byte(buffer, '\t');
return;
}
case 'v': {
parser->current.end++;
pm_buffer_append_u8(buffer, '\v');
pm_buffer_append_byte(buffer, '\v');
return;
}
case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': {
@ -6474,7 +6474,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, uint8_t flags) {
}
}
pm_buffer_append_u8(buffer, value);
pm_buffer_append_byte(buffer, value);
return;
}
case 'x': {
@ -6496,7 +6496,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, uint8_t flags) {
if (flags & PM_ESCAPE_FLAG_REGEXP) {
pm_buffer_append_bytes(buffer, start, (size_t) (parser->current.end - start));
} else {
pm_buffer_append_u8(buffer, value);
pm_buffer_append_byte(buffer, value);
}
} else {
pm_parser_err_current(parser, PM_ERR_ESCAPE_INVALID_HEXADECIMAL);
@ -6690,14 +6690,14 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, uint8_t flags) {
case '\r': {
if (peek_offset(parser, 1) == '\n') {
parser->current.end += 2;
pm_buffer_append_u8(buffer, '\n');
pm_buffer_append_byte(buffer, '\n');
return;
}
}
/* fallthrough */
default: {
if (parser->current.end < parser->end) {
pm_buffer_append_u8(buffer, *parser->current.end++);
pm_buffer_append_byte(buffer, *parser->current.end++);
}
return;
}
@ -6951,7 +6951,7 @@ typedef struct {
// Push the given byte into the token buffer.
static inline void
pm_token_buffer_push(pm_token_buffer_t *token_buffer, uint8_t byte) {
pm_buffer_append_u8(&token_buffer->buffer, byte);
pm_buffer_append_byte(&token_buffer->buffer, byte);
}
// When we're about to return from lexing the current token and we know for sure
@ -15592,14 +15592,14 @@ pm_parse(pm_parser_t *parser) {
PRISM_EXPORTED_FUNCTION void
pm_serialize(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
pm_buffer_append_str(buffer, "PRISM", 5);
pm_buffer_append_u8(buffer, PRISM_VERSION_MAJOR);
pm_buffer_append_u8(buffer, PRISM_VERSION_MINOR);
pm_buffer_append_u8(buffer, PRISM_VERSION_PATCH);
pm_buffer_append_u8(buffer, PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS ? 1 : 0);
pm_buffer_append_string(buffer, "PRISM", 5);
pm_buffer_append_byte(buffer, PRISM_VERSION_MAJOR);
pm_buffer_append_byte(buffer, PRISM_VERSION_MINOR);
pm_buffer_append_byte(buffer, PRISM_VERSION_PATCH);
pm_buffer_append_byte(buffer, PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS ? 1 : 0);
pm_serialize_content(parser, node, buffer);
pm_buffer_append_str(buffer, "\0", 1);
pm_buffer_append_string(buffer, "\0", 1);
}
// Parse and serialize the AST represented by the given source to the given

View File

@ -7,86 +7,134 @@
#include "prism/util/pm_buffer.h"
static void
prettyprint_location(pm_buffer_t *buffer, pm_parser_t *parser, pm_location_t *location) {
prettyprint_node(pm_buffer_t *output_buffer, pm_parser_t *parser, pm_node_t *node, pm_buffer_t *prefix_buffer);
static void
prettyprint_location(pm_buffer_t *output_buffer, pm_parser_t *parser, pm_location_t *location) {
char printed[] = "[0000-0000]";
snprintf(printed, sizeof(printed), "[%04ld-%04ld]", (long int)(location->start - parser->start), (long int)(location->end - parser->start));
pm_buffer_append_str(buffer, printed, strlen(printed));
snprintf(printed, sizeof(printed), "[%04ld-%04ld]", (long int) (location->start - parser->start), (long int) (location->end - parser->start));
pm_buffer_append_string(output_buffer, printed, strlen(printed));
}
static void
prettyprint_node(pm_buffer_t *buffer, pm_parser_t *parser, pm_node_t *node) {
prettyprint_node(pm_buffer_t *output_buffer, pm_parser_t *parser, pm_node_t *node, pm_buffer_t *prefix_buffer) {
switch (PM_NODE_TYPE(node)) {
// We do not need to print a ScopeNode as it's not part
// of the AST
case PM_SCOPE_NODE:
// We do not need to print a ScopeNode as it's not part of the AST.
return;
<%- nodes.each do |node| -%>
case <%= node.type %>: {
pm_buffer_append_str(buffer, "<%= node.name %>(", <%= node.name.length + 1 %>);
<%- if node.fields.any? { |field| !field.is_a?(Prism::FlagsField) } -%>
pm_<%= node.human %>_t *cast = (pm_<%= node.human %>_t *) node;
<%- end -%>
pm_buffer_append_string(output_buffer, "@ <%= node.name %> (location: ()-())\n", <%= node.name.length + 21 %>);
<%- node.fields.each_with_index do |field, index| -%>
<%= "pm_buffer_append_str(buffer, \", \", 2);" if index != 0 -%>
<%- pointer, preadd = index == node.fields.length - 1 ? ["└── ", " "] : ["├── ", "│ "] -%>
// <%= field.name %>
{
pm_buffer_concat(output_buffer, prefix_buffer);
pm_buffer_append_string(output_buffer, "<%= pointer %><%= field.name %>:", <%= pointer.bytesize + field.name.length + 1 %>);
<%- case field -%>
<%- when Prism::NodeField -%>
prettyprint_node(buffer, parser, (pm_node_t *)((pm_<%= node.human %>_t *)node)-><%= field.name %>);
pm_buffer_append_byte(output_buffer, '\n');
size_t prefix_length = prefix_buffer->length;
pm_buffer_append_string(prefix_buffer, "<%= preadd %>", <%= preadd.bytesize %>);
pm_buffer_concat(output_buffer, prefix_buffer);
prettyprint_node(output_buffer, parser, (pm_node_t *) cast-><%= field.name %>, prefix_buffer);
prefix_buffer->length = prefix_length;
<%- when Prism::OptionalNodeField -%>
if (((pm_<%= node.human %>_t *)node)-><%= field.name %> == NULL) {
pm_buffer_append_str(buffer, "nil", 3);
} else {
prettyprint_node(buffer, parser, (pm_node_t *)((pm_<%= node.human %>_t *)node)-><%= field.name %>);
}
if (cast-><%= field.name %> == NULL) {
pm_buffer_append_string(output_buffer, " ∅\n", 5);
} else {
pm_buffer_append_byte(output_buffer, '\n');
size_t prefix_length = prefix_buffer->length;
pm_buffer_append_string(prefix_buffer, "<%= preadd %>", <%= preadd.bytesize %>);
pm_buffer_concat(output_buffer, prefix_buffer);
prettyprint_node(output_buffer, parser, (pm_node_t *) cast-><%= field.name %>, prefix_buffer);
prefix_buffer->length = prefix_length;
}
<%- when Prism::StringField -%>
pm_buffer_append_str(buffer, "\"", 1);
pm_buffer_append_bytes(buffer, pm_string_source(&((pm_<%= node.human %>_t *)node)-><%= field.name %>), pm_string_length(&((pm_<%= node.human %>_t *)node)-><%= field.name %>));
pm_buffer_append_str(buffer, "\"", 1);
pm_buffer_append_string(output_buffer, " \"", 2);
pm_buffer_append_bytes(output_buffer, pm_string_source(&cast-><%= field.name %>), pm_string_length(&cast-><%= field.name %>));
pm_buffer_append_string(output_buffer, "\"\n", 2);
<%- when Prism::NodeListField -%>
pm_buffer_append_str(buffer, "[", 1);
for (uint32_t index = 0; index < ((pm_<%= node.human %>_t *)node)-><%= field.name %>.size; index++) {
if (index != 0) pm_buffer_append_str(buffer, ", ", 2);
prettyprint_node(buffer, parser, (pm_node_t *) ((pm_<%= node.human %>_t *) node)-><%= field.name %>.nodes[index]);
}
pm_buffer_append_str(buffer, "]", 1);
pm_buffer_append_string(output_buffer, " (length: )\n", 13);
size_t last_index = cast-><%= field.name %>.size - 1;
for (uint32_t index = 0; index <= last_index; index++) {
size_t prefix_length = prefix_buffer->length;
pm_buffer_append_string(prefix_buffer, " ", 4);
pm_buffer_concat(output_buffer, prefix_buffer);
if (index == last_index) {
pm_buffer_append_string(output_buffer, "└── ", 10);
pm_buffer_append_string(prefix_buffer, " ", 4);
} else {
pm_buffer_append_string(output_buffer, "├── ", 10);
pm_buffer_append_string(prefix_buffer, "│ ", 6);
}
prettyprint_node(output_buffer, parser, (pm_node_t *) cast-><%= field.name %>.nodes[index], prefix_buffer);
prefix_buffer->length = prefix_length;
}
pm_buffer_append_string(output_buffer, "\n", 1);
<%- when Prism::ConstantField -%>
char <%= field.name %>_buffer[12];
snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "%u", ((pm_<%= node.human %>_t *)node)-><%= field.name %>);
pm_buffer_append_str(buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer));
char <%= field.name %>_buffer[12];
snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "%u", cast-><%= field.name %>);
pm_buffer_append_byte(output_buffer, ' ');
pm_buffer_append_string(output_buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer));
pm_buffer_append_byte(output_buffer, '\n');
<%- when Prism::OptionalConstantField -%>
if (((pm_<%= node.human %>_t *)node)-><%= field.name %> == 0) {
pm_buffer_append_str(buffer, "nil", 3);
} else {
char <%= field.name %>_buffer[12];
snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "%u", ((pm_<%= node.human %>_t *)node)-><%= field.name %>);
pm_buffer_append_str(buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer));
}
if (cast-><%= field.name %> == 0) {
pm_buffer_append_string(output_buffer, " ∅\n", 5);
} else {
char <%= field.name %>_buffer[12];
snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "%u", cast-><%= field.name %>);
pm_buffer_append_byte(output_buffer, ' ');
pm_buffer_append_string(output_buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer));
pm_buffer_append_byte(output_buffer, '\n');
}
<%- when Prism::ConstantListField -%>
pm_buffer_append_str(buffer, "[", 1);
for (uint32_t index = 0; index < ((pm_<%= node.human %>_t *)node)-><%= field.name %>.size; index++) {
if (index != 0) pm_buffer_append_str(buffer, ", ", 2);
char <%= field.name %>_buffer[12];
snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "%u", ((pm_<%= node.human %>_t *)node)-><%= field.name %>.ids[index]);
pm_buffer_append_str(buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer));
}
pm_buffer_append_str(buffer, "]", 1);
pm_buffer_append_string(output_buffer, " [", 2);
for (uint32_t index = 0; index < cast-><%= field.name %>.size; index++) {
if (index != 0) pm_buffer_append_string(output_buffer, ", ", 2);
char <%= field.name %>_buffer[12];
snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "%u", cast-><%= field.name %>.ids[index]);
pm_buffer_append_string(output_buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer));
}
pm_buffer_append_string(output_buffer, "]\n", 2);
<%- when Prism::LocationField -%>
prettyprint_location(buffer, parser, &((pm_<%= node.human %>_t *)node)-><%= field.name %>);
pm_buffer_append_byte(output_buffer, ' ');
prettyprint_location(output_buffer, parser, &cast-><%= field.name %>);
pm_buffer_append_byte(output_buffer, '\n');
<%- when Prism::OptionalLocationField -%>
if (((pm_<%= node.human %>_t *)node)-><%= field.name %>.start == NULL) {
pm_buffer_append_str(buffer, "nil", 3);
} else {
prettyprint_location(buffer, parser, &((pm_<%= node.human %>_t *)node)-><%= field.name %>);
}
if (cast-><%= field.name %>.start == NULL) {
pm_buffer_append_string(output_buffer, " ∅\n", 5);
} else {
pm_buffer_append_byte(output_buffer, ' ');
prettyprint_location(output_buffer, parser, &cast-><%= field.name %>);
pm_buffer_append_byte(output_buffer, '\n');
}
<%- when Prism::UInt32Field -%>
char <%= field.name %>_buffer[12];
snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "+%d", ((pm_<%= node.human %>_t *)node)-><%= field.name %>);
pm_buffer_append_str(buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer));
char <%= field.name %>_buffer[12];
snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "+%d", cast-><%= field.name %>);
pm_buffer_append_byte(output_buffer, ' ');
pm_buffer_append_string(output_buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer));
pm_buffer_append_byte(output_buffer, '\n');
<%- when Prism::FlagsField -%>
char <%= field.name %>_buffer[12];
snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "+%d", (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
pm_buffer_append_str(buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer));
char <%= field.name %>_buffer[12];
snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "+%d", (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
pm_buffer_append_byte(output_buffer, ' ');
pm_buffer_append_string(output_buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer));
pm_buffer_append_byte(output_buffer, '\n');
<%- else -%>
<%- raise -%>
<%- end -%>
}
<%- end -%>
pm_buffer_append_str(buffer, ")", 1);
break;
}
<%- end -%>
@ -95,17 +143,20 @@ prettyprint_node(pm_buffer_t *buffer, pm_parser_t *parser, pm_node_t *node) {
void
pm_print_node(pm_parser_t *parser, pm_node_t *node) {
pm_buffer_t buffer;
if (!pm_buffer_init(&buffer)) return;
pm_buffer_t output_buffer = { 0 };
pm_buffer_t prefix_buffer = { 0 };
prettyprint_node(&buffer, parser, node);
printf("%.*s\n", (int) buffer.length, buffer.value);
prettyprint_node(&output_buffer, parser, node, &prefix_buffer);
printf("%.*s\n", (int) output_buffer.length, output_buffer.value);
pm_buffer_free(&buffer);
pm_buffer_free(&output_buffer);
pm_buffer_free(&prefix_buffer);
}
// Pretty-prints the AST represented by the given node to the given buffer.
PRISM_EXPORTED_FUNCTION void
pm_prettyprint(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
prettyprint_node(buffer, parser, node);
pm_prettyprint(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *output_buffer) {
pm_buffer_t prefix_buffer = { 0 };
prettyprint_node(output_buffer, parser, node, &prefix_buffer);
pm_buffer_free(&prefix_buffer);
}

View File

@ -20,24 +20,24 @@ pm_serialize_location(pm_parser_t *parser, pm_location_t *location, pm_buffer_t
assert(location->end);
assert(location->start <= location->end);
pm_buffer_append_u32(buffer, pm_ptrdifft_to_u32(location->start - parser->start));
pm_buffer_append_u32(buffer, pm_ptrdifft_to_u32(location->end - location->start));
pm_buffer_append_varint(buffer, pm_ptrdifft_to_u32(location->start - parser->start));
pm_buffer_append_varint(buffer, pm_ptrdifft_to_u32(location->end - location->start));
}
static void
pm_serialize_string(pm_parser_t *parser, pm_string_t *string, pm_buffer_t *buffer) {
switch (string->type) {
case PM_STRING_SHARED: {
pm_buffer_append_u8(buffer, 1);
pm_buffer_append_u32(buffer, pm_ptrdifft_to_u32(pm_string_source(string) - parser->start));
pm_buffer_append_u32(buffer, pm_sizet_to_u32(pm_string_length(string)));
pm_buffer_append_byte(buffer, 1);
pm_buffer_append_varint(buffer, pm_ptrdifft_to_u32(pm_string_source(string) - parser->start));
pm_buffer_append_varint(buffer, pm_sizet_to_u32(pm_string_length(string)));
break;
}
case PM_STRING_OWNED:
case PM_STRING_CONSTANT: {
uint32_t length = pm_sizet_to_u32(pm_string_length(string));
pm_buffer_append_u8(buffer, 2);
pm_buffer_append_u32(buffer, length);
pm_buffer_append_byte(buffer, 2);
pm_buffer_append_varint(buffer, length);
pm_buffer_append_bytes(buffer, pm_string_source(string), length);
break;
}
@ -49,7 +49,7 @@ pm_serialize_string(pm_parser_t *parser, pm_string_t *string, pm_buffer_t *buffe
void
pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
pm_buffer_append_u8(buffer, (uint8_t) PM_NODE_TYPE(node));
pm_buffer_append_byte(buffer, (uint8_t) PM_NODE_TYPE(node));
size_t offset = buffer->length;
@ -66,7 +66,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
// serialize length
// encoding of location u32s make us need to save this offset.
size_t length_offset = buffer->length;
pm_buffer_append_str(buffer, "\0\0\0\0", 4); /* consume 4 bytes, updated below */
pm_buffer_append_string(buffer, "\0\0\0\0", 4); /* consume 4 bytes, updated below */
<%- end -%>
<%- node.fields.each do |field| -%>
<%- case field -%>
@ -74,7 +74,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
pm_serialize_node(parser, (pm_node_t *)((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
<%- when Prism::OptionalNodeField -%>
if (((pm_<%= node.human %>_t *)node)-><%= field.name %> == NULL) {
pm_buffer_append_u8(buffer, 0);
pm_buffer_append_byte(buffer, 0);
} else {
pm_serialize_node(parser, (pm_node_t *)((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
}
@ -82,17 +82,17 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
pm_serialize_string(parser, &((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
<%- when Prism::NodeListField -%>
uint32_t <%= field.name %>_size = pm_sizet_to_u32(((pm_<%= node.human %>_t *)node)-><%= field.name %>.size);
pm_buffer_append_u32(buffer, <%= field.name %>_size);
pm_buffer_append_varint(buffer, <%= field.name %>_size);
for (uint32_t index = 0; index < <%= field.name %>_size; index++) {
pm_serialize_node(parser, (pm_node_t *) ((pm_<%= node.human %>_t *)node)-><%= field.name %>.nodes[index], buffer);
}
<%- when Prism::ConstantField, Prism::OptionalConstantField -%>
pm_buffer_append_u32(buffer, pm_sizet_to_u32(((pm_<%= node.human %>_t *)node)-><%= field.name %>));
pm_buffer_append_varint(buffer, pm_sizet_to_u32(((pm_<%= node.human %>_t *)node)-><%= field.name %>));
<%- when Prism::ConstantListField -%>
uint32_t <%= field.name %>_size = pm_sizet_to_u32(((pm_<%= node.human %>_t *)node)-><%= field.name %>.size);
pm_buffer_append_u32(buffer, <%= field.name %>_size);
pm_buffer_append_varint(buffer, <%= field.name %>_size);
for (uint32_t index = 0; index < <%= field.name %>_size; index++) {
pm_buffer_append_u32(buffer, pm_sizet_to_u32(((pm_<%= node.human %>_t *)node)-><%= field.name %>.ids[index]));
pm_buffer_append_varint(buffer, pm_sizet_to_u32(((pm_<%= node.human %>_t *)node)-><%= field.name %>.ids[index]));
}
<%- when Prism::LocationField -%>
<%- if field.should_be_serialized? -%>
@ -101,16 +101,16 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
<%- when Prism::OptionalLocationField -%>
<%- if field.should_be_serialized? -%>
if (((pm_<%= node.human %>_t *)node)-><%= field.name %>.start == NULL) {
pm_buffer_append_u8(buffer, 0);
pm_buffer_append_byte(buffer, 0);
} else {
pm_buffer_append_u8(buffer, 1);
pm_buffer_append_byte(buffer, 1);
pm_serialize_location(parser, &((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
}
<%- end -%>
<%- when Prism::UInt32Field -%>
pm_buffer_append_u32(buffer, ((pm_<%= node.human %>_t *)node)-><%= field.name %>);
pm_buffer_append_varint(buffer, ((pm_<%= node.human %>_t *)node)-><%= field.name %>);
<%- when Prism::FlagsField -%>
pm_buffer_append_u32(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
pm_buffer_append_varint(buffer, (uint32_t)(node->flags & ~PM_NODE_FLAG_COMMON_MASK));
<%- else -%>
<%- raise -%>
<%- end -%>
@ -129,16 +129,16 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
static void
pm_serialize_comment(pm_parser_t *parser, pm_comment_t *comment, pm_buffer_t *buffer) {
// serialize type
pm_buffer_append_u8(buffer, (uint8_t) comment->type);
pm_buffer_append_byte(buffer, (uint8_t) comment->type);
// serialize location
pm_buffer_append_u32(buffer, pm_ptrdifft_to_u32(comment->start - parser->start));
pm_buffer_append_u32(buffer, pm_ptrdifft_to_u32(comment->end - comment->start));
pm_buffer_append_varint(buffer, pm_ptrdifft_to_u32(comment->start - parser->start));
pm_buffer_append_varint(buffer, pm_ptrdifft_to_u32(comment->end - comment->start));
}
static void
pm_serialize_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer) {
pm_buffer_append_u32(buffer, pm_sizet_to_u32(pm_list_size(list)));
pm_buffer_append_varint(buffer, pm_sizet_to_u32(pm_list_size(list)));
pm_comment_t *comment;
for (comment = (pm_comment_t *) list->head; comment != NULL; comment = (pm_comment_t *) comment->node.next) {
@ -149,17 +149,17 @@ pm_serialize_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buf
static void
pm_serialize_magic_comment(pm_parser_t *parser, pm_magic_comment_t *magic_comment, pm_buffer_t *buffer) {
// serialize key location
pm_buffer_append_u32(buffer, pm_ptrdifft_to_u32(magic_comment->key_start - parser->start));
pm_buffer_append_u32(buffer, pm_sizet_to_u32(magic_comment->key_length));
pm_buffer_append_varint(buffer, pm_ptrdifft_to_u32(magic_comment->key_start - parser->start));
pm_buffer_append_varint(buffer, pm_sizet_to_u32(magic_comment->key_length));
// serialize value location
pm_buffer_append_u32(buffer, pm_ptrdifft_to_u32(magic_comment->value_start - parser->start));
pm_buffer_append_u32(buffer, pm_sizet_to_u32(magic_comment->value_length));
pm_buffer_append_varint(buffer, pm_ptrdifft_to_u32(magic_comment->value_start - parser->start));
pm_buffer_append_varint(buffer, pm_sizet_to_u32(magic_comment->value_length));
}
static void
pm_serialize_magic_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer) {
pm_buffer_append_u32(buffer, pm_sizet_to_u32(pm_list_size(list)));
pm_buffer_append_varint(buffer, pm_sizet_to_u32(pm_list_size(list)));
pm_magic_comment_t *magic_comment;
for (magic_comment = (pm_magic_comment_t *) list->head; magic_comment != NULL; magic_comment = (pm_magic_comment_t *) magic_comment->node.next) {
@ -171,17 +171,17 @@ static void
pm_serialize_diagnostic(pm_parser_t *parser, pm_diagnostic_t *diagnostic, pm_buffer_t *buffer) {
// serialize message
size_t message_length = strlen(diagnostic->message);
pm_buffer_append_u32(buffer, pm_sizet_to_u32(message_length));
pm_buffer_append_str(buffer, diagnostic->message, message_length);
pm_buffer_append_varint(buffer, pm_sizet_to_u32(message_length));
pm_buffer_append_string(buffer, diagnostic->message, message_length);
// serialize location
pm_buffer_append_u32(buffer, pm_ptrdifft_to_u32(diagnostic->start - parser->start));
pm_buffer_append_u32(buffer, pm_ptrdifft_to_u32(diagnostic->end - diagnostic->start));
pm_buffer_append_varint(buffer, pm_ptrdifft_to_u32(diagnostic->start - parser->start));
pm_buffer_append_varint(buffer, pm_ptrdifft_to_u32(diagnostic->end - diagnostic->start));
}
static void
pm_serialize_diagnostic_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer) {
pm_buffer_append_u32(buffer, pm_sizet_to_u32(pm_list_size(list)));
pm_buffer_append_varint(buffer, pm_sizet_to_u32(pm_list_size(list)));
pm_diagnostic_t *diagnostic;
for (diagnostic = (pm_diagnostic_t *) list->head; diagnostic != NULL; diagnostic = (pm_diagnostic_t *) diagnostic->node.next) {
@ -192,8 +192,8 @@ pm_serialize_diagnostic_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *
static void
pm_serialize_encoding(pm_encoding_t *encoding, pm_buffer_t *buffer) {
size_t encoding_length = strlen(encoding->name);
pm_buffer_append_u32(buffer, pm_sizet_to_u32(encoding_length));
pm_buffer_append_str(buffer, encoding->name, encoding_length);
pm_buffer_append_varint(buffer, pm_sizet_to_u32(encoding_length));
pm_buffer_append_string(buffer, encoding->name, encoding_length);
}
#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
@ -213,7 +213,7 @@ pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer)
pm_buffer_append_zeroes(buffer, 4);
// Next, encode the length of the constant pool.
pm_buffer_append_u32(buffer, parser->constant_pool.size);
pm_buffer_append_varint(buffer, parser->constant_pool.size);
// Now we're going to serialize the content of the node.
pm_serialize_node(parser, node, buffer);
@ -268,10 +268,10 @@ static void
serialize_token(void *data, pm_parser_t *parser, pm_token_t *token) {
pm_buffer_t *buffer = (pm_buffer_t *) data;
pm_buffer_append_u32(buffer, token->type);
pm_buffer_append_u32(buffer, pm_ptrdifft_to_u32(token->start - parser->start));
pm_buffer_append_u32(buffer, pm_ptrdifft_to_u32(token->end - token->start));
pm_buffer_append_u32(buffer, parser->lex_state);
pm_buffer_append_varint(buffer, token->type);
pm_buffer_append_varint(buffer, pm_ptrdifft_to_u32(token->start - parser->start));
pm_buffer_append_varint(buffer, pm_ptrdifft_to_u32(token->end - token->start));
pm_buffer_append_varint(buffer, parser->lex_state);
}
PRISM_EXPORTED_FUNCTION void
@ -288,7 +288,7 @@ pm_lex_serialize(const uint8_t *source, size_t size, const char *filepath, pm_bu
pm_node_t *node = pm_parse(&parser);
// Append 0 to mark end of tokens
pm_buffer_append_u8(buffer, 0);
pm_buffer_append_byte(buffer, 0);
pm_serialize_encoding(&parser.encoding, buffer);
pm_serialize_comment_list(&parser, &parser.comment_list, buffer);
@ -316,7 +316,7 @@ pm_parse_lex_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer,
parser.lex_callback = &lex_callback;
pm_node_t *node = pm_parse(&parser);
pm_buffer_append_u8(buffer, 0);
pm_buffer_append_byte(buffer, 0);
pm_serialize(&parser, node, buffer);
pm_node_destroy(&parser, node);

View File

@ -40,9 +40,13 @@ pm_buffer_append_length(pm_buffer_t *buffer, size_t length) {
size_t next_length = buffer->length + length;
if (next_length > buffer->capacity) {
do {
if (buffer->capacity == 0) {
buffer->capacity = 1;
}
while (next_length > buffer->capacity) {
buffer->capacity *= 2;
} while (next_length > buffer->capacity);
}
buffer->value = realloc(buffer->value, buffer->capacity);
}
@ -53,20 +57,22 @@ pm_buffer_append_length(pm_buffer_t *buffer, size_t length) {
// Append a generic pointer to memory to the buffer.
static inline void
pm_buffer_append(pm_buffer_t *buffer, const void *source, size_t length) {
size_t cursor = buffer->length;
pm_buffer_append_length(buffer, length);
memcpy(buffer->value + (buffer->length - length), source, length);
memcpy(buffer->value + cursor, source, length);
}
// Append the given amount of space as zeroes to the buffer.
void
pm_buffer_append_zeroes(pm_buffer_t *buffer, size_t length) {
size_t cursor = buffer->length;
pm_buffer_append_length(buffer, length);
memset(buffer->value + (buffer->length - length), 0, length);
memset(buffer->value + cursor, 0, length);
}
// Append a string to the buffer.
void
pm_buffer_append_str(pm_buffer_t *buffer, const char *value, size_t length) {
pm_buffer_append_string(pm_buffer_t *buffer, const char *value, size_t length) {
pm_buffer_append(buffer, value, length);
}
@ -78,27 +84,35 @@ pm_buffer_append_bytes(pm_buffer_t *buffer, const uint8_t *value, size_t length)
// Append a single byte to the buffer.
void
pm_buffer_append_u8(pm_buffer_t *buffer, uint8_t value) {
pm_buffer_append_byte(pm_buffer_t *buffer, uint8_t value) {
const void *source = &value;
pm_buffer_append(buffer, source, sizeof(uint8_t));
}
// Append a 32-bit unsigned integer to the buffer.
// Append a 32-bit unsigned integer to the buffer as a variable-length integer.
void
pm_buffer_append_u32(pm_buffer_t *buffer, uint32_t value) {
pm_buffer_append_varint(pm_buffer_t *buffer, uint32_t value) {
if (value < 128) {
pm_buffer_append_u8(buffer, (uint8_t) value);
pm_buffer_append_byte(buffer, (uint8_t) value);
} else {
uint32_t n = value;
while (n >= 128) {
pm_buffer_append_u8(buffer, (uint8_t) (n | 128));
pm_buffer_append_byte(buffer, (uint8_t) (n | 128));
n >>= 7;
}
pm_buffer_append_u8(buffer, (uint8_t) n);
pm_buffer_append_byte(buffer, (uint8_t) n);
}
}
// Free the memory associated with the buffer.
// Concatenate one buffer onto another.
void
pm_buffer_concat(pm_buffer_t *destination, const pm_buffer_t *source) {
if (source->length > 0) {
pm_buffer_append(destination, source->value, source->length);
}
}
// Free the internal memory associated with the buffer.
void
pm_buffer_free(pm_buffer_t *buffer) {
free(buffer->value);

View File

@ -13,9 +13,9 @@
// block of memory. It is used to store the serialized representation of a
// prism tree.
typedef struct {
char *value;
size_t length;
size_t capacity;
char *value;
} pm_buffer_t;
// Return the size of the pm_buffer_t struct.
@ -37,16 +37,19 @@ PRISM_EXPORTED_FUNCTION size_t pm_buffer_length(pm_buffer_t *buffer);
void pm_buffer_append_zeroes(pm_buffer_t *buffer, size_t length);
// Append a string to the buffer.
void pm_buffer_append_str(pm_buffer_t *buffer, const char *value, size_t length);
void pm_buffer_append_string(pm_buffer_t *buffer, const char *value, size_t length);
// Append a list of bytes to the buffer.
void pm_buffer_append_bytes(pm_buffer_t *buffer, const uint8_t *value, size_t length);
// Append a single byte to the buffer.
void pm_buffer_append_u8(pm_buffer_t *buffer, uint8_t value);
void pm_buffer_append_byte(pm_buffer_t *buffer, uint8_t value);
// Append a 32-bit unsigned integer to the buffer.
void pm_buffer_append_u32(pm_buffer_t *buffer, uint32_t value);
void pm_buffer_append_varint(pm_buffer_t *buffer, uint32_t value);
// Append one buffer onto another.
void pm_buffer_concat(pm_buffer_t *destination, const pm_buffer_t *source);
// Free the memory associated with the buffer.
PRISM_EXPORTED_FUNCTION void pm_buffer_free(pm_buffer_t *buffer);