From bfe24ef83f14a15cd43efafe77ba3f9c0a616168 Mon Sep 17 00:00:00 2001 From: Koichi ITO Date: Wed, 14 Feb 2024 22:13:04 +0900 Subject: [PATCH] Support multi-versioning for `Prism::Translation::Parser` ## Summary Fixes #2356. I'm working on integrating Prism into RuboCop. This PR introduces `Prism::Translation::Parser33` and `Prism::Translation::Parser34`, named in accordance with the following comments. https://github.com/rubocop/rubocop/issues/12600#issuecomment-1932707748 Currently, `Prism::Translation::Parser` always operates in Ruby 3.4 mode. This means it will not parse as Ruby 3.3 even if `TargetRubyVersion: 80_82_73_83_77.33` is specified. Therefore, the `it` introduced in Ruby 3.4 is parsed incompatibly with Ruby 3.3. In Ruby 3.3, the expected name for an `lvar` is `:it`, not `:"0it"`. ### Expected AST The following is an expected AST when parsing Ruby 3.3 code: ```console $ bundle exec ruby -rprism -rprism/translation/parser33 -ve "p Prism::Translation::Parser33.parse('items.map { it.do_something }')" ruby 3.3.0 (2023-12-25 revision 5124f9ac75) [x86_64-darwin22] s(:block, s(:send, s(:send, nil, :items), :map), s(:args), s(:send, s(:send, nil, :it), :do_something)) ``` ### Actual AST The following is an actual AST when parsing Ruby 3.3 code: ```console $ ruby -rprism -ve "p Prism::Translation::Parser.parse('items.map { it.do_something }')" ruby 3.3.0 (2023-12-25 revision 5124f9ac75) [x86_64-darwin22] s(:block, s(:send, s(:send, nil, :items), :map), s(:args), s(:send, s(:lvar, :"0it"), :do_something)) ``` `Prism::Translation::Parser33` and `Prism::Translation::Parser34` aim to correspond to Ruby 3.3 and Ruby 3.4, respectively. And, The hack of specifying `TargetRubyVersion: 80_82_73_83_77.33` is expected to become unnecessary in the future, but the behavior will be maintained until RuboCop's support is finalized: https://github.com/rubocop/rubocop/issues/12600#issuecomment-1933657732 ## Additional Information A private method named `convert_for_prism` is prepared to convert the `version` from Parser to the `version` expected by Prism. For example, a Parser-compatible value is `3.3`, whereas Prism expects `"3.3.0"`. `Parser#version` is not used in RuboCop, but it's unclear how it is utilized in other libraries that rely on the Parser gem. Therefore, logic to maintain compatibility between Parser and Prism is implemented. --- lib/prism/translation/parser.rb | 18 +++++++++++++++--- lib/prism/translation/parser/rubocop.rb | 8 ++++++-- lib/prism/translation/parser33.rb | 12 ++++++++++++ lib/prism/translation/parser34.rb | 12 ++++++++++++ prism.gemspec | 2 ++ 5 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 lib/prism/translation/parser33.rb create mode 100644 lib/prism/translation/parser34.rb diff --git a/lib/prism/translation/parser.rb b/lib/prism/translation/parser.rb index 6e678dde6b4..0354e8195ba 100644 --- a/lib/prism/translation/parser.rb +++ b/lib/prism/translation/parser.rb @@ -43,7 +43,7 @@ def parse(source_buffer) source = source_buffer.source offset_cache = build_offset_cache(source) - result = unwrap(Prism.parse(source, filepath: source_buffer.name), offset_cache) + result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version)), offset_cache) build_ast(result.value, offset_cache) ensure @@ -56,7 +56,7 @@ def parse_with_comments(source_buffer) source = source_buffer.source offset_cache = build_offset_cache(source) - result = unwrap(Prism.parse(source, filepath: source_buffer.name), offset_cache) + result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version)), offset_cache) [ build_ast(result.value, offset_cache), @@ -75,7 +75,7 @@ def tokenize(source_buffer, recover = false) offset_cache = build_offset_cache(source) result = begin - unwrap(Prism.parse_lex(source, filepath: source_buffer.name), offset_cache) + unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version)), offset_cache) rescue ::Parser::SyntaxError raise if !recover end @@ -168,6 +168,18 @@ def build_range(location, offset_cache) ) end + # Converts the version format handled by Parser to the format handled by Prism. + def convert_for_prism(version) + case version + when 33 + "3.3.0" + when 34 + "3.4.0" + else + "latest" + end + end + require_relative "parser/compiler" require_relative "parser/lexer" diff --git a/lib/prism/translation/parser/rubocop.rb b/lib/prism/translation/parser/rubocop.rb index 3e34fc7ace6..32349478e89 100644 --- a/lib/prism/translation/parser/rubocop.rb +++ b/lib/prism/translation/parser/rubocop.rb @@ -12,6 +12,7 @@ class Parser # This is the special version number that should be used in rubocop # configuration files to trigger using prism. VERSION_3_3 = 80_82_73_83_77.33 + VERSION_3_4 = 80_82_73_83_77.34 # This module gets prepended into RuboCop::AST::ProcessedSource. module ProcessedSource @@ -19,8 +20,11 @@ module ProcessedSource # list of known parsers. def parser_class(ruby_version) if ruby_version == Prism::Translation::Parser::VERSION_3_3 - require "prism/translation/parser" - Prism::Translation::Parser + require "prism/translation/parser33" + Prism::Translation::Parser33 + if ruby_version == Prism::Translation::Parser::VERSION_3_4 + require "prism/translation/parser34" + Prism::Translation::Parser34 else super end diff --git a/lib/prism/translation/parser33.rb b/lib/prism/translation/parser33.rb new file mode 100644 index 00000000000..cfaa623163a --- /dev/null +++ b/lib/prism/translation/parser33.rb @@ -0,0 +1,12 @@ +require_relative "parser" + +module Prism + module Translation + # This class is the entry-point for Ruby 3.3 of `Prism::Translation::Parser`. + class Parser33 < Parser + def version # :nodoc: + 33 + end + end + end +end diff --git a/lib/prism/translation/parser34.rb b/lib/prism/translation/parser34.rb new file mode 100644 index 00000000000..0a34758659c --- /dev/null +++ b/lib/prism/translation/parser34.rb @@ -0,0 +1,12 @@ +require_relative "parser" + +module Prism + module Translation + # This class is the entry-point for Ruby 3.4 of `Prism::Translation::Parser`. + class Parser34 < Parser + def version # :nodoc: + 34 + end + end + end +end diff --git a/prism.gemspec b/prism.gemspec index b32627a0dff..ec00ad484a7 100644 --- a/prism.gemspec +++ b/prism.gemspec @@ -87,6 +87,8 @@ Gem::Specification.new do |spec| "lib/prism/serialize.rb", "lib/prism/translation.rb", "lib/prism/translation/parser.rb", + "lib/prism/translation/parser33.rb", + "lib/prism/translation/parser34.rb", "lib/prism/translation/parser/compiler.rb", "lib/prism/translation/parser/lexer.rb", "lib/prism/translation/parser/rubocop.rb",