1berry/ruby
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Switch the default parser from parse.y to Prism

Browse Source 
This commit switches the default parser to Prism. There are a
couple of additional changes related to this that are a part of
this as well to make this happen.

* Switch the default parser in parse.h
* Remove the Prism-specific workflow and add a parse.y-specific
  workflow to CI so that it continues to be tested
* Update a few test exclusions since Prism has the correct
  behavior but parse.y doesn't per
  https://bugs.ruby-lang.org/issues/20504.
* Skips a couple of tests on RBS which are failing because they
  are using RubyVM::AbstractSyntaxTree.of.

Fixes [Feature #20564]
This commit is contained in:
Kevin Newton 2024-08-28 16:35:03 -04:00
parent f2919bd11c
commit ea2af5782d
Notes: git 2024-09-12 17:43:28 +00:00 
Merged: https://github.com/ruby/ruby/pull/11497
15 changed files with 68 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/bundled_gems.yml vendored
View File

@ -103,7 +103,7 @@ jobs:
timeout-minutes: 30
env:
RUBY_TESTOPTS: '-q --tty=no'
TEST_BUNDLED_GEMS_ALLOW_FAILURES: ''
TEST_BUNDLED_GEMS_ALLOW_FAILURES: 'typeprof'
if: ${{ steps.diff.outputs.gems }}
- name: Commit

2
.github/workflows/macos.yml vendored
View File

@ -138,7 +138,7 @@ jobs:
timeout-minutes: 60
env:
RUBY_TESTOPTS: '-q --tty=no'
TEST_BUNDLED_GEMS_ALLOW_FAILURES: ''
TEST_BUNDLED_GEMS_ALLOW_FAILURES: 'typeprof'
PRECHECK_BUNDLED_GEMS: 'no'
- name: make skipped tests

21
.github/workflows/prism.yml → .github/workflows/parsey.yml vendored
View File

@ -1,4 +1,4 @@
name: Prism
name: parse.y
on:
push:
paths-ignore:
@ -33,31 +33,31 @@ jobs:
matrix:
include:
- test_task: test
run_opts: '--parser=prism'
run_opts: '--parser=parse.y'
testopts: '-v --tty=no'
timeout: 30
- test_task: test-all
run_opts: '--parser=prism'
testopts: '-q --tty=no --excludes-dir="../src/test/.excludes-prism" --exclude="error_highlight/test_error_highlight.rb"'
run_opts: '--parser=parse.y'
testopts: '-q --tty=no --excludes-dir="../src/test/.excludes-parsey"'
timeout: 40
- test_task: test-spec
run_opts: '--parser=prism'
specopts: '-T --parser=prism'
run_opts: '--parser=parse.y'
specopts: '-T --parser=parse.y'
timeout: 10
- test_task: test-tool
run_opts: '--parser=prism'
run_opts: '--parser=parse.y'
testopts: '-v --tty=no'
timeout: 30
- test_task: test-bundler-parallel
run_opts: '--parser=prism'
run_opts: '--parser=parse.y'
testopts: '-v --tty=no'
timeout: 50
- test_task: test-bundled-gems
run_opts: '--parser=prism'
run_opts: '--parser=parse.y'
testopts: '-v --tty=no'
timeout: 30
- test_task: test-syntax-suggest
run_opts: '--parser=prism'
run_opts: '--parser=parse.y'
testopts: '-v --tty=no'
timeout: 30
fail-fast: false
@ -105,6 +105,7 @@ jobs:
RUBY_TESTOPTS: ${{ matrix.testopts }}
RUN_OPTS: ${{ matrix.run_opts }}
SPECOPTS: ${{ matrix.specopts }}
TEST_BUNDLED_GEMS_ALLOW_FAILURES: 'typeprof'
- uses: ./.github/actions/slack
with:

2
.github/workflows/ubuntu.yml vendored
View File

@ -144,7 +144,7 @@ jobs:
timeout-minutes: ${{ matrix.timeout || 40 }}
env:
RUBY_TESTOPTS: '-q --tty=no'
TEST_BUNDLED_GEMS_ALLOW_FAILURES: ''
TEST_BUNDLED_GEMS_ALLOW_FAILURES: 'typeprof'
PRECHECK_BUNDLED_GEMS: 'no'
- name: make skipped tests

2
.github/workflows/yjit-macos.yml vendored
View File

@ -134,7 +134,7 @@ jobs:
timeout-minutes: 60
env:
RUBY_TESTOPTS: '-q --tty=no'
TEST_BUNDLED_GEMS_ALLOW_FAILURES: ''
TEST_BUNDLED_GEMS_ALLOW_FAILURES: 'typeprof'
SYNTAX_SUGGEST_TIMEOUT: '5'
PRECHECK_BUNDLED_GEMS: 'no'
continue-on-error: ${{ matrix.continue-on-test_task || false }}

2
.github/workflows/yjit-ubuntu.yml vendored
View File

@ -187,7 +187,7 @@ jobs:
timeout-minutes: 90
env:
RUBY_TESTOPTS: '-q --tty=no'
TEST_BUNDLED_GEMS_ALLOW_FAILURES: ''
TEST_BUNDLED_GEMS_ALLOW_FAILURES: 'typeprof'
PRECHECK_BUNDLED_GEMS: 'no'
SYNTAX_SUGGEST_TIMEOUT: '5'
YJIT_BINDGEN_DIFF_OPTS: '--exit-code'

2
internal/parse.h
View File

@ -16,7 +16,7 @@
// 0: parse.y
// 1: Prism
#ifndef RB_DEFAULT_PARSER
#define RB_DEFAULT_PARSER 0
#define RB_DEFAULT_PARSER 1
#endif
#ifdef UNIVERSAL_PARSER

37
prism_compile.c
View File

@ -1399,10 +1399,12 @@ pm_compile_hash_elements(rb_iseq_t *iseq, const pm_node_t *node, const pm_node_l
switch (PM_NODE_TYPE(element)) {
case PM_ASSOC_NODE: {
// Pre-allocation check (this branch can be omitted).
if (PM_NODE_FLAG_P(element, PM_NODE_FLAG_STATIC_LITERAL) && (
(!static_literal && ((index + min_tmp_hash_length) < elements->size)) ||
(first_chunk && stack_length == 0)
)) {
if (
PM_NODE_FLAG_P(element, PM_NODE_FLAG_STATIC_LITERAL) && (
(!static_literal && ((index + min_tmp_hash_length) < elements->size)) ||
(first_chunk && stack_length == 0)
)
) {
// Count the elements that are statically-known.
size_t count = 1;
while (index + count < elements->size && PM_NODE_FLAG_P(elements->nodes[index + count], PM_NODE_FLAG_STATIC_LITERAL)) count++;
@ -5640,6 +5642,14 @@ pm_compile_constant_path_operator_write_node(rb_iseq_t *iseq, const pm_constant_
PUSH_INSN1(ret, location, setconstant, name);
}
/**
* Many nodes in Prism can be marked as a static literal, which means slightly
* different things depending on which node it is. Occasionally we need to omit
* container nodes from static literal checks, which is where this macro comes
* in.
*/
#define PM_CONTAINER_P(node) (PM_NODE_TYPE_P(node, PM_ARRAY_NODE) || PM_NODE_TYPE_P(node, PM_HASH_NODE) || PM_NODE_TYPE_P(node, PM_RANGE_NODE))
/**
* Compiles a prism node into instruction sequences.
*
@ -5867,12 +5877,21 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret,
new_array_size++;
}
}
else if (PM_NODE_FLAG_P(element, PM_NODE_FLAG_STATIC_LITERAL) && !static_literal && ((index + min_tmp_array_size) < elements->size)) {
else if (
PM_NODE_FLAG_P(element, PM_NODE_FLAG_STATIC_LITERAL) &&
!PM_CONTAINER_P(element) &&
!static_literal &&
((index + min_tmp_array_size) < elements->size)
) {
// If we have a static literal, then there's the potential
// to group a bunch of them together with a literal array
// and then concat them together.
size_t right_index = index + 1;
while (right_index < elements->size && PM_NODE_FLAG_P(elements->nodes[right_index], PM_NODE_FLAG_STATIC_LITERAL)) right_index++;
while (
right_index < elements->size &&
PM_NODE_FLAG_P(elements->nodes[right_index], PM_NODE_FLAG_STATIC_LITERAL) &&
!PM_CONTAINER_P(elements->nodes[right_index])
) right_index++;
size_t tmp_array_size = right_index - index;
if (tmp_array_size >= min_tmp_array_size) {
@ -8975,7 +8994,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret,
pm_node_t *value = cast->value;
name = cast->name;
if (PM_NODE_FLAG_P(value, PM_NODE_FLAG_STATIC_LITERAL) && !(PM_NODE_TYPE_P(value, PM_ARRAY_NODE) || PM_NODE_TYPE_P(value, PM_HASH_NODE) || PM_NODE_TYPE_P(value, PM_RANGE_NODE))) {
if (PM_NODE_FLAG_P(value, PM_NODE_FLAG_STATIC_LITERAL) && !PM_CONTAINER_P(value)) {
rb_ary_push(default_values, pm_static_literal_value(iseq, value, scope_node));
}
else {
@ -9285,7 +9304,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret,
pm_node_t *value = cast->value;
name = cast->name;
if (!PM_NODE_FLAG_P(value, PM_NODE_FLAG_STATIC_LITERAL) || PM_NODE_TYPE_P(value, PM_ARRAY_NODE) || PM_NODE_TYPE_P(value, PM_HASH_NODE) || PM_NODE_TYPE_P(value, PM_RANGE_NODE)) {
if (!PM_NODE_FLAG_P(value, PM_NODE_FLAG_STATIC_LITERAL) || PM_CONTAINER_P(value)) {
LABEL *end_label = NEW_LABEL(location.line);
pm_local_index_t index = pm_lookup_local_index(iseq, scope_node, name, 0);
@ -9815,6 +9834,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret,
}
}
#undef PM_CONTAINER_P
/** True if the given iseq can have pre execution blocks. */
static inline bool
pm_iseq_pre_execution_p(rb_iseq_t *iseq)

0
test/.excludes-prism/TestM17N.rb → test/.excludes-parsey/TestM17N.rb
View File

0
test/.excludes-prism/TestMixedUnicodeEscape.rb → test/.excludes-parsey/TestMixedUnicodeEscape.rb
View File

0
test/.excludes-prism/TestRubyLiteral.rb → test/.excludes-parsey/TestRubyLiteral.rb
View File

5
test/ruby/test_literal.rb
View File

@ -241,8 +241,9 @@ class TestRubyLiteral < Test::Unit::TestCase
def test_dregexp
assert_instance_of Regexp, /re#{'ge'}xp/
assert_equal(/regexp/, /re#{'ge'}xp/)
bug3903 = '[ruby-core:32682]'
assert_raise(SyntaxError, bug3903) {eval('/[#{"\x80"}]/')}
# [ruby-core:32682]
eval('/[#{"\x80"}]/')
end
def test_array

8
test/ruby/test_m17n.rb
View File

@ -1439,20 +1439,20 @@ class TestM17N < Test::Unit::TestCase
assert_regexp_usascii_literal('//', Encoding::US_ASCII)
assert_regexp_usascii_literal('/#{ }/', Encoding::US_ASCII)
assert_regexp_usascii_literal('/#{"a"}/', Encoding::US_ASCII)
assert_regexp_usascii_literal('/#{%q"\x80"}/', Encoding::ASCII_8BIT)
assert_regexp_usascii_literal('/#{"\x80"}/', nil, SyntaxError)
assert_regexp_usascii_literal('/#{%q"\x80"}/', Encoding::US_ASCII)
assert_regexp_usascii_literal('/#{"\x80"}/', Encoding::ASCII_8BIT)
assert_regexp_usascii_literal('/a/', Encoding::US_ASCII)
assert_regexp_usascii_literal('/a#{ }/', Encoding::US_ASCII)
assert_regexp_usascii_literal('/a#{"a"}/', Encoding::US_ASCII)
assert_regexp_usascii_literal('/a#{%q"\x80"}/', Encoding::ASCII_8BIT)
assert_regexp_usascii_literal('/a#{"\x80"}/', nil, SyntaxError)
assert_regexp_usascii_literal('/a#{"\x80"}/', Encoding::ASCII_8BIT)
assert_regexp_usascii_literal('/\x80/', Encoding::ASCII_8BIT)
assert_regexp_usascii_literal('/\x80#{ }/', Encoding::ASCII_8BIT)
assert_regexp_usascii_literal('/\x80#{"a"}/', Encoding::ASCII_8BIT)
assert_regexp_usascii_literal('/\x80#{%q"\x80"}/', Encoding::ASCII_8BIT)
assert_regexp_usascii_literal('/\x80#{"\x80"}/', nil, SyntaxError)
assert_regexp_usascii_literal('/\x80#{"\x80"}/', Encoding::ASCII_8BIT)
assert_regexp_usascii_literal('/\u1234/', Encoding::UTF_8)
assert_regexp_usascii_literal('/\u1234#{ }/', Encoding::UTF_8)

2
test/ruby/test_mixed_unicode_escapes.rb
View File

@ -18,12 +18,12 @@ class TestMixedUnicodeEscape < Test::Unit::TestCase
assert_raise(SyntaxError) { eval %q("\u{1234}é<>")}
# also should not work for Regexp
assert_raise(SyntaxError) { eval %q(/#{"\u1234"}#{"é<>"}/)}
assert_raise(RegexpError) { eval %q(/\u{1234}#{nil}é<>/)}
assert_raise(RegexpError) { eval %q(/é<>#{nil}\u1234/)}
# String interpolation turns into an expression and we get
# a different kind of error, but we still can't mix these
assert_raise(Encoding::CompatibilityError) { eval %q(/#{"\u1234"}#{"é<>"}/)}
assert_raise(Encoding::CompatibilityError) { eval %q("\u{1234}#{nil}é<>")}
assert_raise(Encoding::CompatibilityError) { eval %q("é<>#{nil}\u1234")}
end

14
tool/rbs_skip_tests
View File

@ -63,3 +63,17 @@ TestInstanceNetHTTPResponse depending on external resources
test_initialize(TempfileRemoverSingletonTest) Remover class is removed at https://github.com/ruby/tempfile/commit/753ab16642fdc3cf92e1cf3dd1d80e8f75128735
test_call(TempfileRemoverTest) Remover class is removed at https://github.com/ruby/tempfile/commit/753ab16642fdc3cf92e1cf3dd1d80e8f75128735
# Switching to prism resulted in an unexpected error, `RuntimeError: cannot get AST for ISEQ compiled by prism`, raised by `RubyVM::AbstractSyntax.of(method)`
test_1(RBS::RuntimePrototypeTest)
test_decls_for_anonymous_class_or_module(RBS::RuntimePrototypeTest)
test_decls_structure(RBS::RuntimePrototypeTest)
test_define_method_alias(RBS::RuntimePrototypeTest)
test_for_initialize_type(RBS::RuntimePrototypeTest)
test_for_overwritten_module_name(RBS::RuntimePrototypeTest)
test_for_yield(RBS::RuntimePrototypeTest)
test_include_owner(RBS::RuntimePrototypeTest)
test_merge_types(RBS::RuntimePrototypeTest)
test_reflection(RBS::RuntimePrototypeTest)
test_todo(RBS::RuntimePrototypeTest)
test_of(RubyVM::AbstractSyntaxTreeSingletonTest)