diff --git a/.rspec b/.rspec new file mode 100644 index 0000000..c99d2e7 --- /dev/null +++ b/.rspec @@ -0,0 +1 @@ +--require spec_helper diff --git a/.rubocop.yml b/.rubocop.yml index 82299da..9ea596e 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -8,7 +8,7 @@ AllCops: - ruby - rake SuggestExtensions: false - TargetRubyVersion: 2.1 + TargetRubyVersion: 2.6 Lint: Enabled: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 0742c6a..5c4573b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,35 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +The minimum ruby version is now 2.6. + +### Added + +- up-to-date unicode properties map, e.g. for `\p{Dash}` + +### Fixed + +- capture numbering in atomic and possessive groups + * e.g. `(?>a(b))\1` +- backreferences and conditionals after optional or repeated captures + * e.g. `(a)?\1` +- handling of backreference numbers followed by space and digits in x-mode + * e.g. `/(a)\1 0/x` +- capture choices and repeat limits when expanding conditionals + * e.g. `(a){0}(?(1)b|c)` +- greedy and lazy matching order when expanding repeated groups + * e.g. `(?:(aa)|(a))+?\1\2` +- handling of repeated keep marks (`\K`) and root alternatives + * e.g. `a\Kb\Kc|d` +- repeat limits and zero-width patterns in absence groups + * e.g. `(?~(?=a))` +- handling of astral characters in some Unicode types and properties + * e.g. `(?u:\d)` matching `𝟡` +- handling of Unicode numbers in word boundaries + * e.g. `\b½\b` +- missing escaping of Unicode line separators in regex literal output + * e.g. `Regexp.new("\u2028")` + ## [3.14.0] - 2025-08-03 ### Added diff --git a/README.md b/README.md index 7a623b4..f72d8f3 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,8 @@ Many Regexp tokens work in JavaScript just as they do in Ruby, or allow for a st The tricky bit here is that these expressions may be nested, and that their expansions may increase the capturing group count. This means that any following backreferences need an update. E.g. (.{3})\g<1>(.)\2 (which matches strings like "FooBarXX") converts to (.{3})(.{3})(.)\3. +**Non-participating capturing groups** are another tricky case and require complex branching in the pattern. See #26 for details. + ## Contributions Feel free to send suggestions, point out issues, or submit pull requests. diff --git a/js_regex.gemspec b/js_regex.gemspec index 9d359f6..8b87755 100644 --- a/js_regex.gemspec +++ b/js_regex.gemspec @@ -19,7 +19,7 @@ Gem::Specification.new do |s| s.files = Dir['lib/**/*.{csv,rb}'] - s.required_ruby_version = '>= 2.1.0' + s.required_ruby_version = '>= 2.6.0' s.add_dependency 'character_set', '~> 1.4' s.add_dependency 'regexp_parser', '~> 2.11' diff --git a/lib/js_regex/converter/anchor_converter.rb b/lib/js_regex/converter/anchor_converter.rb index d375088..fff22fb 100644 --- a/lib/js_regex/converter/anchor_converter.rb +++ b/lib/js_regex/converter/anchor_converter.rb @@ -51,9 +51,9 @@ def convert_nonboundary # This is an approximation to the word boundary behavior in Ruby, c.f. # https://github.com/ruby/ruby/blob/08476c45/tool/enc-unicode.rb#L130 - W = '\d\p{L}\p{M}\p{Pc}' + W = '\p{N}\p{L}\p{M}\p{Pc}' BOUNDARY_EXPANSION = "(?:(?<=[#{W}])(?=[^#{W}]|$)|(?<=[^#{W}]|^)(?=[#{W}]))" - NONBOUNDARY_EXPANSION = "(?<=[#{W}])(?=[#{W}])" + NONBOUNDARY_EXPANSION = "(?:(?<=[#{W}])(?=[#{W}])|(?<=[^#{W}]|^)(?=[^#{W}]|$))" def pass_boundary_with_warning warn_of("The anchor '#{data}' at index #{expression.ts} only works "\ diff --git a/lib/js_regex/converter/backreference_converter.rb b/lib/js_regex/converter/backreference_converter.rb index 84b2ba0..c7124ed 100644 --- a/lib/js_regex/converter/backreference_converter.rb +++ b/lib/js_regex/converter/backreference_converter.rb @@ -32,28 +32,26 @@ def convert_name_ref end def convert_to_plain_num_ref - position = new_position - - # Check if this backreference refers to a group that was recursively called - original_group = target_position - if (recursive_position = context.get_recursive_group_position(original_group)) - # Use the position of the group created by the recursive call - position = recursive_position + positions = context.get_recursive_group_positions(target_position) + positions = [new_position] if positions.empty? + references = positions.map do |position| + Node.new("\\#{position}", reference: position, type: :backref) + end + if references.length > 1 && Utils::Captures.nonempty?(expression.referenced_expression) + Node.new(*references, type: :capture_backref) + else + references.first end - - text = "\\#{position}#{'(?:)' if expression.x?}" - Node.new(text, reference: position, type: :backref) end def convert_multiplexed_name_ref # Create alternation of all groups with the same name - positions = expression.referenced_expressions.map do |ref_exp| - context.new_capturing_group_position(ref_exp.number) + # Ruby tries groups with the same name from last to first. + references = expression.referenced_expressions.reverse.map do |ref_exp| + position = context.new_capturing_group_position(ref_exp.number) + Node.new("\\#{position}", reference: position, type: :backref) end - - # Build alternation like (?:\1|\2) - alternation = positions.map { |pos| "\\#{pos}" }.join('|') - Node.new("(?:#{alternation})") + Node.new(*references, type: :multiplex_backref) end def new_position @@ -72,48 +70,15 @@ def convert_call context.count_recursion(expression) - # Track groups before the wrapper group is added - groups_before_wrapper = context.capturing_group_count - - context.increment_local_capturing_group_count - target_copy = expression.referenced_expression.unquantified_clone - # avoid "Duplicate capture group name" error in JS - target_copy.token = :capture if target_copy.is?(:named, :group) context.start_subexp_recursion - result = convert_expression(target_copy) + ref = expression.referenced_expression + result = Converter.for(ref).convert(ref, context, quantified: false) context.end_subexp_recursion - # Track all groups created during this recursive call - # This handles both the directly called group and any nested groups within it - # Get all group numbers from the referenced expression - original_groups = collect_group_numbers(expression.referenced_expression) - - # The first new group number is groups_before_wrapper + 1 - # (the wrapper group from increment_local_capturing_group_count doesn't appear in output) - first_new_group = groups_before_wrapper + 1 - - # Map each original group to its corresponding new group - # For example, if we recursively called group 1 which contains group 2, - # and this created groups 3 and 4, then: - # - group 1 -> group 3 - # - group 2 -> group 4 - original_groups.each_with_index do |old_group_num, index| - new_group_num = first_new_group + index - context.track_recursive_group_call(old_group_num, new_group_num) - end - # wrap in passive group if it is a full-pattern recursion expression.reference == 0 ? Node.new('(?:', result, ')') : result end - def collect_group_numbers(exp) - return [] if exp.terminal? - - numbers = [] - numbers << exp.number if exp.capturing? - exp.each_expression { |sub| numbers += collect_group_numbers(sub) } - numbers - end end end end diff --git a/lib/js_regex/converter/base.rb b/lib/js_regex/converter/base.rb index 01284ff..f09865a 100644 --- a/lib/js_regex/converter/base.rb +++ b/lib/js_regex/converter/base.rb @@ -11,13 +11,15 @@ module Converter # class Base # returns instance of Node with #quantifier attached. - def convert(expression, context) + def convert(expression, context, quantified: true) self.context = context self.expression = expression - node = convert_data - node = Node.new(node) if node.instance_of?(String) - apply_quantifier(node) + if quantified && expression.quantifier&.possessive? + wrap_in_backrefed_lookahead { convert_node(quantified) } + else + convert_node(quantified) + end end private @@ -33,19 +35,17 @@ def data end alias pass_through data - def apply_quantifier(node) - return node if node.dropped? || (qtf = expression.quantifier).nil? + def convert_node(quantified) + node = convert_data + node = Node.new(node) if node.instance_of?(String) + quantified ? apply_quantifier(node) : node + end - if qtf.possessive? - node.update(quantifier: qtf.dup.tap { |q| q.text = q.text[0..-2] }) - return wrap_in_backrefed_lookahead(node) - elsif qtf.token == :interval && qtf.text[0..1] == "{," - node.update(quantifier: qtf.dup.tap { |q| q.text = "{0,#{q.max}}" }) - else - node.update(quantifier: qtf) - end + def apply_quantifier(node) + return node if node.dropped? || + (qtf = Node::Quantifier.from_expression(expression)).nil? - node + node.update(quantifier: qtf) end def convert_subexpressions @@ -70,11 +70,7 @@ def warn_of_unsupported_feature(description = nil, min_target: nil) end def warn_of(text) - if context.fail_fast - raise ConversionError, text.sub(/^Dropped /, '') - else - context.warnings << text - end + context.warn_of(text) end def drop @@ -82,13 +78,16 @@ def drop end alias drop_without_warning drop - def wrap_in_backrefed_lookahead(content) - number = context.capturing_group_count + 1 + def wrap_in_backrefed_lookahead + number = context.capture_group(added: true) + content = yield + if content.is_a?(Node) && content.dropped? + context.cancel_added_capturing_group + return content + end backref_node = Node.new("\\#{number}", reference: number, type: :backref) backrefed_group = Node.new('(', *content, ')', reference: number, type: :captured_group) - context.increment_local_capturing_group_count - # The surrounding group is added so that quantifiers apply to the whole. - # Without it, `(?:)` would need to be appended as literal digits may follow. + # The surrounding group makes quantifiers apply to the whole expansion. Node.new('(?:(?=', backrefed_group, ')', backref_node, ')') end diff --git a/lib/js_regex/converter/conditional_converter.rb b/lib/js_regex/converter/conditional_converter.rb index 0696305..5ae2b55 100644 --- a/lib/js_regex/converter/conditional_converter.rb +++ b/lib/js_regex/converter/conditional_converter.rb @@ -18,11 +18,17 @@ def convert_data end def mark_conditional_for_second_pass - reference = expression.referenced_expression.number - node = Node.new('(?:', reference: reference, type: :conditional) - expression.branches.each do |branch| - node << Node.new('(?:', convert_expression(branch), ')') + target = expression.referenced_expression + positions = context.get_recursive_group_positions(target.number) + branches = expression.branches.map { |branch| Node.new('(?:', convert_expression(branch), ')') } + if positions.length > 1 && Utils::Captures.nonempty?(target) + references = positions.map { |position| Node.new(reference: position) } + return Node.new(Node.new(*references), *branches, type: :capture_conditional) end + + reference = positions.first || context.new_capturing_group_position(target.number) + node = Node.new('(?:', reference: reference, type: :conditional) + branches.each { |branch| node << branch } node << ')' end end diff --git a/lib/js_regex/converter/context.rb b/lib/js_regex/converter/context.rb index e5ade7f..e66961a 100644 --- a/lib/js_regex/converter/context.rb +++ b/lib/js_regex/converter/context.rb @@ -24,11 +24,20 @@ def initialize(case_insensitive_root: false, fail_fast: false, target: nil) self.required_options_hash = {} self.warnings = [] self.recursive_group_map = {} + self.recursive_group_history = Hash.new { |hash, key| hash[key] = [] } self.case_insensitive_root = case_insensitive_root self.target = target end + def warn_of(text) + if fail_fast + raise ConversionError, text.sub(/^Dropped /, '') + else + warnings << text + end + end + # target context def es_2015_or_higher? @@ -57,7 +66,10 @@ def required_options # group context - def capture_group + def capture_group(added: false) + if added + added_capturing_groups_after_group[original_capturing_group_count] += 1 + end self.capturing_group_count = capturing_group_count + 1 end @@ -69,9 +81,9 @@ def end_atomic_group self.in_atomic_group = false end - def increment_local_capturing_group_count - added_capturing_groups_after_group[original_capturing_group_count] += 1 - capture_group + def cancel_added_capturing_group + added_capturing_groups_after_group[original_capturing_group_count] -= 1 + self.capturing_group_count = capturing_group_count - 1 end def recursions(exp) @@ -93,14 +105,14 @@ def start_subexp_recursion end def end_subexp_recursion - self.in_subexp_recursion = false # Pop the last recursion from stack when exiting recursion_stack.pop if recursion_stack.any? + self.in_subexp_recursion = recursion_stack.any? end # Get the number of groups at the start of the current recursion def recursion_start_group_count - self.recursion_start_group_count || 0 + @recursion_start_group_count || 0 end # takes and returns 1-indexed group positions. @@ -120,6 +132,9 @@ def original_capturing_group_count # Track that a group was created by a recursive call def track_recursive_group_call(original_group_num, new_group_num) recursive_group_map[original_group_num] = new_group_num + history = recursive_group_history[original_group_num] + history.delete(new_group_num) + history.unshift(new_group_num) end # Get the group number created by a recursive call @@ -127,6 +142,10 @@ def get_recursive_group_position(original_group_num) recursive_group_map[original_group_num] end + def get_recursive_group_positions(original_group_num) + recursive_group_history[original_group_num].dup + end + private attr_accessor :added_capturing_groups_after_group, @@ -134,6 +153,7 @@ def get_recursive_group_position(original_group_num) :recursion_stack, :required_options_hash, :recursive_group_map, + :recursive_group_history, :target attr_writer :capturing_group_count, @@ -145,7 +165,7 @@ def get_recursive_group_position(original_group_num) :warnings def total_added_capturing_groups - added_capturing_groups_after_group.values.inject(0, &:+) + added_capturing_groups_after_group.values.sum end end end diff --git a/lib/js_regex/converter/group_converter.rb b/lib/js_regex/converter/group_converter.rb index 3530db1..5833d7b 100644 --- a/lib/js_regex/converter/group_converter.rb +++ b/lib/js_regex/converter/group_converter.rb @@ -36,7 +36,7 @@ def emulate_atomic_group build_passive_group else context.start_atomic_group - result = wrap_in_backrefed_lookahead(convert_subexpressions) + result = wrap_in_backrefed_lookahead { convert_subexpressions } context.end_atomic_group result end @@ -70,8 +70,10 @@ def unmatchable_absence_group? end def build_absence_group - head = "(?:(?:.|\\n){,#{expression.inner_match_length.min - 1}}|(?:(?!" - tail = ')(?:.|\n))*)' + head = '(?:(?:(?!' + maximum = expression.inner_match_length.min - 1 + suffix = maximum > 0 ? "(?:.|\\n){0,#{maximum}}" : '' + tail = ")(?:.|\\n))*#{suffix})" build_group(head: head, tail: tail, capturing: false) end @@ -80,9 +82,13 @@ def build_group(opts = {}) tail = opts[:tail] || ')' return Node.new(*wrap(head, tail)) if opts[:capturing].equal?(false) - context.capture_group unless context.in_subexp_recursion - ref = expression.number - Node.new(*wrap(head, tail), reference: ref, type: :captured_group) + ref = context.capture_group(added: context.in_subexp_recursion) + context.track_recursive_group_call(expression.number, ref) + node = Node.new(*wrap(head, tail), reference: ref, type: :captured_group, + marked_nonempty: Utils::Captures.nonempty?(expression), + nullable: Utils::Captures.nullable?(expression)) + context.track_recursive_group_call(expression.number, ref) + node end def wrap(head, tail) diff --git a/lib/js_regex/converter/literal_converter.rb b/lib/js_regex/converter/literal_converter.rb index 4cbc8f1..476c0ad 100644 --- a/lib/js_regex/converter/literal_converter.rb +++ b/lib/js_regex/converter/literal_converter.rb @@ -15,7 +15,7 @@ def convert_data if context.case_insensitive_root && !expression.case_insensitive? warn_of_unsupported_feature('nested case-sensitive literal') elsif !context.case_insensitive_root && expression.case_insensitive? - return handle_locally_case_insensitive_literal(result) + return handle_locally_case_insensitive_literal(data) end result end @@ -23,12 +23,21 @@ def convert_data HAS_CASE_PATTERN = /[\p{lower}\p{upper}]/.freeze def handle_locally_case_insensitive_literal(literal) - literal =~ HAS_CASE_PATTERN ? case_insensitivize(literal) : literal + literal.match?(HAS_CASE_PATTERN) ? case_insensitivize(literal) : Utils::Literals.convert_data(literal, context) end def case_insensitivize(literal) literal.each_char.each_with_object(Node.new) do |chr, node| - node << (chr =~ HAS_CASE_PATTERN ? "[#{chr}#{chr.swapcase}]" : chr) + if chr.match?(HAS_CASE_PATTERN) + if chr.ord > 0xFFFF && !context.enable_u_option + node << Node.new('(?:', Utils::Literals.convert_data(chr, context), '|', + Utils::Literals.convert_data(chr.swapcase, context), ')') + else + node << "[#{chr}#{chr.swapcase}]" + end + else + node << Utils::Literals.convert_data(chr, context) + end end end end diff --git a/lib/js_regex/converter/property_converter.rb b/lib/js_regex/converter/property_converter.rb index 48a1e3c..1525b58 100644 --- a/lib/js_regex/converter/property_converter.rb +++ b/lib/js_regex/converter/property_converter.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require_relative 'base' -require 'character_set' class JsRegex module Converter @@ -20,21 +19,9 @@ def convert_data context.enable_u_option "\\#{expression.negative? ? 'P' : 'p'}{#{prop_name_in_js}}" else - build_character_set + Utils::CharSets.convert(expression, context) end end - - def build_character_set - content = CharacterSet.of_expression(expression) - - if expression.case_insensitive? && !context.case_insensitive_root - content = content.case_insensitive - elsif !expression.case_insensitive? && context.case_insensitive_root - warn_of_unsupported_feature('nested case-sensitive property') - end - - content.to_s_with_surrogate_ranges - end end end end diff --git a/lib/js_regex/converter/set_converter.rb b/lib/js_regex/converter/set_converter.rb index 390cf85..9d7873c 100644 --- a/lib/js_regex/converter/set_converter.rb +++ b/lib/js_regex/converter/set_converter.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require_relative 'base' -require 'character_set' class JsRegex module Converter @@ -17,7 +16,7 @@ class SetConverter < JsRegex::Converter::Base private def convert_data - simple_conversion || full_recalculation + simple_conversion || Utils::CharSets.convert(expression, context) end def simple_conversion @@ -64,7 +63,7 @@ def simple_convert_child(exp) def simple_convert_literal_child(exp) if !context.u? && - exp.text =~ Utils::Literals::ASTRAL_PLANE_CODEPOINT_PATTERN && + exp.text.match?(Utils::Literals::ASTRAL_PLANE_CODEPOINT_PATTERN) && !context.enable_u_option false elsif SET_LITERALS_REQUIRING_ESCAPE_PATTERN.match?(exp.text) @@ -78,31 +77,6 @@ def simple_convert_literal_child(exp) SET_SPECIFIC_ESCAPES_PATTERN = /[\^\-]/.freeze CONVERTIBLE_ESCAPE_TOKENS = %i[control meta_sequence bell escape octal] + Utils::Escapes::ESCAPES_SHARED_BY_RUBY_AND_JS - - def full_recalculation - # Fetch codepoints as if the set was case-sensitive, then re-add - # case-insensitivity if needed. - # This way we preserve the casing of the original set in cases where the - # whole regexp is case-insensitive, e.g. /[ABc]/i => /[ABc]/i. - content = original_case_character_set - if expression.case_insensitive? && !context.case_insensitive_root - content = content.case_insensitive - elsif !expression.case_insensitive? && context.case_insensitive_root - warn_of_unsupported_feature('nested case-sensitive set') - end - if context.es_2015_or_higher? - context.enable_u_option if content.astral_part? - content.to_s(format: 'es6', in_brackets: true) - else - content.to_s_with_surrogate_ranges - end - end - - def original_case_character_set - neutral_set = expression.dup - neutral_set.each_expression(true) { |exp| exp.options[:i] = false } - CharacterSet.of_expression(neutral_set) - end end end end diff --git a/lib/js_regex/converter/type_converter.rb b/lib/js_regex/converter/type_converter.rb index a8aadf1..952cec7 100644 --- a/lib/js_regex/converter/type_converter.rb +++ b/lib/js_regex/converter/type_converter.rb @@ -16,14 +16,10 @@ def convert_data when :nonhex then nonhex_expansion when :linebreak then linebreak_expansion when :xgrapheme then xgrapheme - when :digit, :space, :word + when :digit, :space, :word, :nondigit, :nonspace, :nonword return pass_through if Utils::CharTypes.directly_compatible?(expression) - set_substitution - when :nondigit, :nonspace, :nonword - return pass_through if Utils::CharTypes.directly_compatible?(expression) - - negative_set_substitution + Utils::CharSets.convert(expression, context) else warn_of_unsupported_feature end @@ -58,28 +54,14 @@ def nonhex_expansion ES2018_NONHEX_EXPANSION = '\P{AHex}' def linebreak_expansion - wrap_in_backrefed_lookahead(LINEBREAK_EXPANSION) + wrap_in_backrefed_lookahead { LINEBREAK_EXPANSION } end LINEBREAK_EXPANSION = '(?:\r\n|[\n\v\f\r\u0085\u2028\u2029])' - def negative_set_substitution - # ::of_expression returns an inverted set for negative expressions, - # so we need to un-invert before wrapping in [^ and ]. Kinda lame. - "[^#{character_set.inversion.bmp_part}]" - end - - def set_substitution - character_set.bmp_part.to_s(in_brackets: true) - end - - def character_set - CharacterSet.of_expression(expression) - end - def xgrapheme if context.es_2018_or_higher? && context.enable_u_option - wrap_in_backrefed_lookahead(ES2018_XGRAPHEME_EXPANSION) + wrap_in_backrefed_lookahead { ES2018_XGRAPHEME_EXPANSION } else warn_of_unsupported_feature end diff --git a/lib/js_regex/node.rb b/lib/js_regex/node.rb index cf70095..5ccde99 100644 --- a/lib/js_regex/node.rb +++ b/lib/js_regex/node.rb @@ -7,32 +7,47 @@ class JsRegex # class Node require_relative 'error' + Dir[File.join(__dir__, 'node', '*.rb')].sort.each { |f| require f } + + ATTRIBUTES = %i[ + children + marked_nonempty + nullable + quantifier + reference + type + ].freeze - attr_reader :children, :quantifier, :reference, :type + attr_reader(*ATTRIBUTES) - TYPES = %i[ + REFERENCE_TYPES = %i[ backref + capture_backref + capture_check + capture_conditional + conditional + multiplex_backref + ].freeze + + REFERENCE_TYPES_MAP = REFERENCE_TYPES.to_h { |k| [k, true] }.freeze + + TYPES = (REFERENCE_TYPES + %i[ branch captured_group - conditional dropped keep_mark plain - ].freeze + ]).freeze + + TYPES_MAP = TYPES.to_h { |k| [k, true] }.freeze - def initialize(*children, reference: nil, type: :plain) + def initialize(*children, reference: nil, type: :plain, marked_nonempty: false, nullable: false, quantifier: nil) self.children = children self.reference = reference self.type = type - end - - def initialize_copy(*) - self.children = children.map(&:clone) - end - - def transform(&block) - children.map!(&block) - self + self.marked_nonempty = marked_nonempty + self.nullable = nullable + self.quantifier = quantifier end def <<(node) @@ -40,34 +55,9 @@ def <<(node) self end - def dropped? - # keep everything else, including empty or depleted capturing groups - # so as not to not mess with reference numbers (e.g. backrefs) - type.equal?(:dropped) - end - - def to_s - case type - when :dropped - '' - when :backref, :branch, :captured_group, :plain - children.join << quantifier.to_s - else - raise TypeError.new( - "#{type} must be substituted before stringification" - ).extend(JsRegex::Error) - end - end - - def update(attrs) - self.children = attrs.fetch(:children) if attrs.key?(:children) - self.quantifier = attrs.fetch(:quantifier) if attrs.key?(:quantifier) - self.type = attrs.fetch(:type) if attrs.key?(:type) - self - end - - def optional? - quantifier && quantifier.min == 0 + def initialize_copy(*) + super + self.children = children.map(&:clone) end private @@ -75,7 +65,7 @@ def optional? TypeError = Class.new(::TypeError).extend(JsRegex::Error) def type=(arg) - arg.nil? || TYPES.include?(arg) || + arg.nil? || TYPES_MAP[arg] || raise(TypeError, "unsupported type #{arg.class} for #{__method__}") @type = arg end @@ -87,7 +77,7 @@ def children=(arg) end def quantifier=(arg) - arg.nil? || arg.class == Regexp::Expression::Quantifier || + arg.nil? || arg.class == Quantifier || raise(TypeError, "unsupported type #{arg.class} for #{__method__}") @quantifier = arg end @@ -97,5 +87,17 @@ def reference=(arg) raise(TypeError, "unsupported type #{arg.class} for #{__method__}") @reference = arg end + + def marked_nonempty=(arg) + arg == true || arg == false || + raise(TypeError, "unsupported type #{arg.class} for #{__method__}") + @marked_nonempty = arg + end + + def nullable=(arg) + arg == true || arg == false || + raise(TypeError, "unsupported type #{arg.class} for #{__method__}") + @nullable = arg + end end end diff --git a/lib/js_regex/node/checks.rb b/lib/js_regex/node/checks.rb new file mode 100644 index 0000000..d7c658d --- /dev/null +++ b/lib/js_regex/node/checks.rb @@ -0,0 +1,70 @@ +class JsRegex::Node + def dropped? + # keep everything else, including empty or depleted capturing groups + # so as not to not mess with reference numbers (e.g. backrefs) + type.equal?(:dropped) + end + + def branch? + type == :branch + end + + def alternation? + children.any? && children.all? do |child| + child.is_a?(JsRegex::Node) && child.branch? + end + end + + def repeated? + !!quantifier&.repeated? + end + + def optional? + !!quantifier&.optional? + end + + def zero_or_one? + !!quantifier&.zero_or_one? + end + + def zero_quantified? + !!quantifier&.zero? + end + + def max_one? + !!quantifier&.max_one? + end + + def negative_assertion? + head = children.first + head == '(?!' || head == '(? 1 + end + + def bounded_repetition? + !unbounded? && max > 1 + end + + def minimum_at_most_one? + min <= 1 + end + + def zero_or_more? + optional? && unbounded? + end + + def one_or_more? + min == 1 && unbounded? + end + + def fixed? + min == max + end + + def optional_iteration?(index) + index > min + end + + def remaining_minimum(consumed) + [min - consumed, 0].max + end + + def iteration_indices + raise ArgumentError, 'cannot enumerate unbounded repetitions' if unbounded? + + (1..max) + end + + def before_last_iteration?(index) + index < max + end + + def repeat_word(word) + raise ArgumentError, 'variable repetition has no constant word' unless fixed? + + word * min + end + + # Prefixes before one required iteration, in quantifier preference order. + def prefix_counts + minimum = remaining_minimum(1) + counts = (0...minimum).map { |count| [count, count] } + [[minimum, -1]] + reluctant? ? counts : counts.reverse + end + + def with_minimum_one + self.class.new(1, max, reluctant: reluctant?) + end + + def reluctant? + !!@reluctant + end + + def to_s + case [min, max] + when [0, -1] then '*' + when [1, -1] then '+' + when [0, 1] then '?' + else + if min == max + "{#{min}}" + else + "{#{min},#{max < 0 ? '' : max}}" + end + end + (reluctant? ? '?' : '') + end + + private + + attr_reader :min, :max + + def min=(arg) + arg.instance_of?(Integer) || + raise(TypeError, "unsupported type #{arg.class} for #{__method__}") + @min = arg + end + + def max=(arg) + arg.instance_of?(Integer) || + raise(TypeError, "unsupported type #{arg.class} for #{__method__}") + @max = arg + end + + def reluctant=(arg) + arg == true || arg == false || + raise(TypeError, "unsupported type #{arg.class} for #{__method__}") + @reluctant = arg + end +end diff --git a/lib/js_regex/node/stringification.rb b/lib/js_regex/node/stringification.rb new file mode 100644 index 0000000..a2b7e81 --- /dev/null +++ b/lib/js_regex/node/stringification.rb @@ -0,0 +1,14 @@ +class JsRegex::Node + def to_s + case type + when :dropped + '' + when :backref, :branch, :capture_check, :captured_group, :plain + children.join << quantifier.to_s + else + raise TypeError.new( + "#{type} must be substituted before stringification" + ).extend(JsRegex::Error) + end + end +end diff --git a/lib/js_regex/node/transformations.rb b/lib/js_regex/node/transformations.rb new file mode 100644 index 0000000..8756826 --- /dev/null +++ b/lib/js_regex/node/transformations.rb @@ -0,0 +1,56 @@ +class JsRegex::Node + # Creates a copy with the given attributes replaced, without mutating self. + def with(**attrs) + validate_attrs(**attrs) + children = attrs.key?(:children) ? attrs.delete(:children) : self.children.map(&:clone) + new_attrs = (ATTRIBUTES - [:children]).to_h do |attr| + [attr, attrs.fetch(attr) { send(attr) }] + end + self.class.new(*children, **new_attrs) + end + + # Mutates self with validations. + def update(**attrs) + validate_attrs(**attrs) + ATTRIBUTES.each { |attr| send("#{attr}=", attrs.fetch(attr)) if attrs.key?(attr) } + self + end + + def transform(&block) + children.map!(&block) + self + end + + # Preserve optionality: a failing node with ? can still be skipped. + def make_unmatchable + update(type: :plain, reference: nil, children: ['(?!)']) + end + + # Keep numeric reference text in sync with its annotation. Captures and + # unresolved conditions only store the number in their annotation. + def renumber(number) + self.reference = number + case type + when :backref, :capture_check then update(children: ["\\#{number}"]) + when :captured_group, :conditional then nil + else raise TypeError, "cannot renumber #{type} node" + end + self + end + + def make_passive + raise TypeError, "cannot make #{type} passive" unless type == :captured_group + + update(type: :plain, reference: nil, children: ['(?:', *children[1..-1]]) + end + + def quantify(min, max, reluctant: false) + update(quantifier: Quantifier.new(min, max, reluctant: reluctant)) + end + + private + + def validate_attrs(**attrs) + attrs.each_key { |k| ATTRIBUTES.include?(k) || raise(ArgumentError, "no attribute #{k}") } + end +end diff --git a/lib/js_regex/node/traversal.rb b/lib/js_regex/node/traversal.rb new file mode 100644 index 0000000..b9faa48 --- /dev/null +++ b/lib/js_regex/node/traversal.rb @@ -0,0 +1,24 @@ +class JsRegex::Node + # Traverse the node tree, depth first. + # If `skip_self: true` is given, yield only the children and their descendants. + # If `trace: true` is given, yield the node and its path on both entry and exit. + def walk(skip_self: false, trace: false, path: [], &block) + return enum_for(__method__, skip_self: skip_self, trace: trace, path: path) unless block + + if trace + path << self + yield(self, :enter, path) if !skip_self + else + yield(self) if !skip_self + end + + children.each do |child| + child.walk(trace: trace, path: path, &block) unless child.instance_of?(String) + end + + if trace + yield(self, :exit, path) if !skip_self + path.pop + end + end +end diff --git a/lib/js_regex/second_pass.rb b/lib/js_regex/second_pass.rb index e5cd79f..0fa6988 100644 --- a/lib/js_regex/second_pass.rb +++ b/lib/js_regex/second_pass.rb @@ -1,183 +1,405 @@ # frozen_string_literal: true class JsRegex - # - # After conversion of a full Regexp::Expression tree, this - # checks for Node instances that need further processing. - # - module SecondPass - class << self - def call(tree) - substitute_root_level_keep_mark(tree) - alternate_conditional_permutations(tree) - handle_non_participating_backrefs(tree) - tree - end - - private - - def substitute_root_level_keep_mark(tree) - keep_mark_index = nil - tree.children.each.with_index do |child, i| - break keep_mark_index = i if child.type == :keep_mark - end - return unless keep_mark_index - - pre = tree.children[0...keep_mark_index] - post = tree.children[(keep_mark_index + 1)..-1] - lookbehind = Node.new('(?<=', *pre, ')') - tree.update(children: [lookbehind, *post]) - end - - def handle_non_participating_backrefs(tree) - level = 0 - completed_group_numbers = {} - group_branches = {} - branch_stack = [] - - crawl(tree, true) do |node, event| - case [node.type, event] - when [:branch, :enter] - branch_stack.push(node) - when [:branch, :exit] - branch_stack.pop - when [:captured_group, :enter] - level += 1 - when [:captured_group, :exit] - unless node.optional? # ignore optional groups - group_branches[node.reference] = branch_stack.last - end - number = level - number += 1 while completed_group_numbers[number] - completed_group_numbers[number] = true - level -= 1 - when [:backref, :exit] - ref_branch = group_branches[node.reference] - current_branch = branch_stack.last - - # make bad backrefs non-matchable - references_other_branch = - ref_branch && current_branch && ref_branch != current_branch - forward_reference = !completed_group_numbers[node.reference] - if references_other_branch || forward_reference - node.update(type: :plain, children: ['(?!)']) - end - end - end + # Run ordered rewrites on the converted node tree to preserve Ruby matching + # semantics in JavaScript. Share traversal, capture analysis, and node-building + # helpers with the individual steps. + # Example: /\1(a)/ => /(?!)(a)/, since Ruby rejects the forward reference. + class SecondPass + Node = JsRegex::Node + + Dir[File.join(__dir__, 'second_pass', '*.rb')].sort.each do |file| + require file + end + + # `tree` is a JsRegex::Node. SecondPass and its subclasses mutate it. + def self.call(tree) + new(tree).call + tree + end + + def initialize(tree) + self.tree = tree + end + + def call + # the whole second pass is only needed if the regexp uses specific features. + return unless tree.walk.any? { |node| node.internal_reference? || node.type == :keep_mark } + + STEPS.each { |step| step.call(tree) } + end + + STEPS = [ + SubstituteRootLevelKeepMark, + # Expand bounded loops first so later steps can inspect each iteration. + ExpandBoundedCaptureLoops, + ExpandNullableCaptures, + ExpandConstantCaptureLoops, + PreserveRepeatedCaptures, + # Resolve capture histories after all rewrites that can duplicate groups. + ResolveCaptureReferences, + AlternateConditionalPermutations, + HandleNonParticipatingBackrefs, + # Conditional expansion can renumber references, so delimit them last. + DelimitBackrefs, + ].freeze + + private + + attr_accessor :tree + + # + # Shared helper methods used in multiple subclasses / steps + # + + # Children ['(?:', a, ')'] => 2, the insertion point before the closing ')'. + def sequence_end(node) + return if node.alternation? || node.conditional? + + head = node.children.first + if ['(', '(?:', '(?=', '(?!', '(?<=', '(? {1 => 2}; the capture itself is not counted. + def reference_counts(node) + counts = Hash.new(0) + node.walk { |n| counts[n.reference] += 1 if n.reference && n.type != :captured_group } + counts + end - alternatives = permutations.map.with_index do |variant, i| - Node.new((i.zero? ? '(?:' : '|(?:'), variant, ')', type: :branch) - end - tree.update(children: alternatives) + # Capture 1 at \1 in /(a)\1/ => true; in /(a)?\1/ => false. + def required_on_path?(group_path, path) + group_path&.all? do |ancestor| + !skippable_scope?(ancestor) || path.include?(ancestor) end + end - def conditional_tree_permutations(tree) - conds = conditions(tree) - return [] if conds.empty? + # /(a)?/ => true; /(a)/ => false; a branch of /a|b/ => true. + def skippable_scope?(node) + node.optional? || node.branch? || node.negative_assertion? + end - caps_per_branch = captured_group_count(tree) + # Force a node to occur at least once (unless it is quantified with {0,0}). + # /(a)?/ => /(a)/; /(a)*?/ => /(a){1,}?/. + def min_quantify(node) + return unless node.optional? - condition_permutations(conds).map.with_index do |truthy_conds, i| - tree_permutation = tree.clone - # find referenced groups and conditionals and make one-sided - crawl(tree_permutation) do |node| - build_permutation(node, conds, truthy_conds, caps_per_branch, i) - end - tree_permutation - end + if node.zero_quantified? + node.update(children: ['(?!)', *node.children], quantifier: nil) + elsif node.max_one? + node.update(quantifier: nil) + else + node.update(quantifier: node.quantifier.with_minimum_one) end + end - def crawl(node, trace = false, &block) - return if node.instance_of?(String) + # Discard a node's contents, retaining only disabled capture slots. + # /(a(b))+/ => /(()){0}/; /(?:abc)/ => an empty sequence. + # Plain () would participate; quantifying with {0} keeps it unmatched. + def discard_contents(node) + captures = [] + node.walk(skip_self: true) { |el| captures << empty_capture(el) if el.type == :captured_group } + if node.type == :captured_group + node.update(children: ['(', *captures, ')']) + elsif captures.empty? + return node.update(type: :plain, reference: nil, children: [], quantifier: nil) + else + node.update(type: :plain, reference: nil, children: ['(?:', *captures, ')']) + end + node.quantify(0, 0) + end - trace ? yield(node, :enter) : yield(node) - node.children.each { |child| crawl(child, trace, &block) } - trace && yield(node, :exit) + # A copy of /(a*)+/ => /()/, with the same capture ID and an empty value. + def empty_capture(capture) + Node.new('(', ')', reference: capture.reference, type: :captured_group, nullable: true) + end + + # With capture 1 present: /(?(1)b|(c))/ => /(?:(?:b)(?:()){0})/. + # Keep discarded captures in source order so later references keep their numbers. + def select_conditional_branch(node, truthy) + branches = node.conditional_branches + branches.each.with_index { |branch, i| discard_contents(branch) unless i == (truthy ? 0 : 1) } + node.update(type: :plain, reference: nil, children: ['(?:', *branches, ')']) + end + + # At the end of the input, a backreference succeeds only when its value + # is empty or unmatched. Negating that test checks for a nonempty value. + # (1, true) => /(?=[\s\S]*(?![\s\S])(?!\1))/; + # (1, false) => /(?=[\s\S]*(?![\s\S])\1)/. + def nonempty_capture_check(reference, nonempty) + backref = capture_value(reference) + test = nonempty ? Node.new('(?!', backref, ')') : backref + Node.new('(?=[\\s\\S]*(?![\\s\\S])', test, ')') + end + + # /(a)(b)?/ => [2]; /(?:(a)|b)/ => [1]. + def potentially_absent_groups(node = tree, optional = false, reachable_only: false) + return [] if node.instance_of?(String) || (reachable_only && unmatchable?(node)) + + optional ||= node.optional? || node.conditional? || + node.negative_assertion? + groups = optional && node.type == :captured_group ? [node.reference] : [] + children = node.children + if node.alternation? + children = children.reject { |child| unmatchable?(child) } if reachable_only + optional ||= children.length > 1 + end + children.each do |child| + groups.concat(potentially_absent_groups(child, optional, reachable_only: reachable_only)) end + groups + end - def conditions(tree) - conditions = [] - crawl(tree) do |node| - conditions << node.reference if node.type.equal?(:conditional) - end - conditions + # With a generated failure node: /a(?!)/ => true; /(?!)?/ => false, + # because the failing part can be skipped. + def unmatchable?(node) + return node == '(?!)' if node.instance_of?(String) + return false if node.optional? || node.negative_assertion? + + if node.conditional? + branches = node.conditional_branches + branches.length == 2 && branches.all? { |child| unmatchable?(child) } + elsif node.alternation? + node.children.all? { |child| unmatchable?(child) } + else + node.children.any? { |child| unmatchable?(child) } end + end + + # Make sure the referenced group actually matches by preventing optional + # groups and alternative branches from skipping it. Returns true if the + # referenced group is anywhere inside this node, including the node itself. + # Requiring capture 1: /(?:(a)|b)?/ => /(?:(a)|b(?!))/; returns true. + def require_group_participation(node, reference) + return false if node.instance_of?(String) - def captured_group_count(tree) - count = 0 - crawl(tree) { |node| count += 1 if node.type.equal?(:captured_group) } - count + children_with_group = node.children.map do |child| + require_group_participation(child, reference) end + found = node.type == :captured_group && node.reference == reference + return false unless found || children_with_group.any? - def condition_permutations(conditions) - (0..(conditions.length)).inject([]) do |arr, n| - arr + conditions.combination(n).to_a + min_quantify(node) + if node.children.length > 1 && node.alternation? + node.children.each.with_index do |branch, i| + branch << Node.new('(?!)') unless children_with_group[i] end end + true + end + + # After copying capture 1: /(a)(a)\1/ => /(a)(a)history[2,1]/, + # where "history" is a placeholder for renumbering references later. + # The two groups start with the same annotation and receive distinct numbers. + def renumber_captures + nodes, paths = reference_nodes_with_paths + candidates = renumber_groups(nodes, paths) + nodes.each do |node| + parent = paths[node][-2] + # A capture history replaces its children when it is renumbered. + next if parent&.type == :capture_backref - def build_permutation(node, conds, truthy_conds, caps_per_branch, i) - truthy = truthy_conds.include?(node.reference) - - case node.type - when :backref - # We cannot use named groups or backrefs in the conditional expansion, - # their repetition would cause a "Duplicate capture group name" error in JS. - node.update(children: [ - node.children.first.sub(/k<.*>/, node.reference.to_s) - ]) - # backref numbers need to be incremented for subsequent "branches" - adapt_backref_to_permutation(node, caps_per_branch, i) - when :captured_group - # if the group is referenced by any condition, modulate its quantity - if conds.include?(node.reference) - adapt_referenced_group_to_permutation(node, truthy) + renumber_reference(node, candidates) + end + end + + # Original capture IDs [1, 1, 2] => [1, 2, 3]; candidates for old ID 1 + # are [2, 1], before excluding copies on incompatible paths. + def renumber_groups(nodes, paths) + positions = Hash.new { |hash, key| hash[key] = [] } + capture_paths = {} + nodes.select { |node| node.type == :captured_group }.each.with_index do |node, index| + number = index + 1 + positions[node.reference].unshift(number) + capture_paths[number] = paths[node] + node.renumber(number) + end + lambda do |reference, node| + positions[reference].reject do |number| + capture_path = capture_paths[number] + reference_path = paths[node] + # Captures from a negative assertion are always unset outside it. + mutually_exclusive?(capture_path, reference_path) || capture_path.any? do |ancestor| + ancestor.negative_assertion? && !reference_path.include?(ancestor) end - when :conditional - adapt_conditional_to_permutation(node, truthy) end end + end + + # For /(a)\1/, collect the capture and reference nodes with their paths + # from the root; ordinary literal nodes are omitted. + def reference_nodes_with_paths + nodes = [] + paths = {} + tree.walk(trace: true) do |node, event, path| + next unless event == :enter + next unless node.reference || [:capture_backref, :capture_conditional].include?(node.type) - def adapt_referenced_group_to_permutation(group_node, truthy) - truthy ? min_quantify(group_node) : null_quantify(group_node) + nodes << node + paths[node] = path.dup end + [nodes, paths] + end - def adapt_conditional_to_permutation(conditional_node, truthy) - branches = conditional_node.children[1...-1] - if branches.count == 1 - truthy || null_quantify(branches.first) + # A condition on capture 1, copied into groups 2 and 3, becomes a condition + # on either copy; a backreference becomes history[3,2]. + def renumber_reference(node, candidates) + case node.type + when :backref, :capture_check + renumber_backreference(node, candidates.call(node.reference, node)) + when :conditional + references = candidates.call(node.reference, node) + if references.length == 1 + node.renumber(references.first) else - null_quantify(truthy ? branches.last : branches.first) + checks = Node.new(*references.map { |number| Node.new(reference: number) }) + node.update(type: :capture_conditional, reference: nil, + children: [checks, *node.conditional_branches]) end - conditional_node.update(type: :plain) + when :capture_backref + references = node.children.flat_map { |child| candidates.call(child.reference, node) } + node.update(children: references.map { |number| backref(number) }) + when :capture_conditional + checks = node.children.first + references = checks.children.flat_map { |child| candidates.call(child.reference, node) } + checks.update(children: references.map { |number| Node.new(reference: number) }) end + end - def adapt_backref_to_permutation(backref_node, caps_per_branch, i) - new_num = backref_node.reference + caps_per_branch * i - backref_node.update(children: ["\\#{new_num}"]) + # \1 with candidates [3] => \3; with [3, 2] => history[3,2]. + def renumber_backreference(node, references) + if references.length == 1 + node.renumber(references.first) + elsif references.empty? && node.type == :capture_check + node.update(type: :plain, reference: nil, children: ['']) + else + node.update(type: :capture_backref, reference: nil, + children: references.map { |number| backref(number) }) end + end - def min_quantify(node) - return unless node.optional? + # 1 => a /\1/ node that must fail if capture 1 did not participate. + def backref(number) + Node.new("\\#{number}", reference: number, type: :backref) + end - qtf = node.quantifier - if qtf.max.equal?(1) # any zero_or_one quantifier (?, ??, ?+) - node.update(quantifier: nil) - else - min_quantifier = qtf.dup - min_quantifier.text = "{1,#{qtf.max}}#{'?' if qtf.reluctant?}" - node.update(quantifier: min_quantifier) + # These references inspect a value and permit an unmatched capture. + # 1 => a /\1/ node that may match empty when capture 1 did not participate. + def capture_value(number) + Node.new("\\#{number}", reference: number, type: :capture_check) + end + + # (?(1)a|b) => a condition over capture 1's copies, keeping branches a and b + # for ResolveCaptureReferences to choose using the current capture values. + def make_condition_dynamic(node) + checks = Node.new(Node.new(reference: node.reference)) + node.update(type: :capture_conditional, reference: nil, children: [checks, *node.conditional_branches]) + end + + # [/a/, /b/] => /(?:a|b)/; [] => /(?!)/. + def alternate(branches) + return Node.new('(?!)') if branches.empty? + return branches.first if branches.length == 1 + + parts = branches.map { |branch| (branch.is_a?(Node) && branch.branch?) ? branch : Node.new(branch, type: :branch) } + parts.each.with_index { |part, index| part.children.unshift('|') unless index.zero? } + Node.new('(?:', Node.new(*parts), ')') + end + + # /(?:a|b)/ => [/(?:a)/, /(?:b)/], preserving wrappers on each choice. + def split_alternatives(node) + if node.alternation? + node.children.map do |branch| + chosen = branch.with(type: :plain) + chosen.children.shift if chosen.children.first == '|' + chosen + end + else + grouped = ['(', '(?:'].include?(node.children.first) && node.children.last == ')' + return unless grouped || node.children.length == 1 + + children = node.children.grep(Node) + return unless children.length == 1 && !children.first.quantifier + + alternatives = split_alternatives(children.first) + return unless alternatives + + alternatives.map do |alternative| + node.with(children: node.children.map { |el| el.equal?(children.first) ? alternative : el.clone }) end end + end + + # Body /a/ => /(a)/ with a fresh negative capture ID, numbered later. + def temporary_capture(*children) + # Negative IDs avoid existing group numbers until renumber_captures runs. + @temporary_reference ||= -1 + Node.new('(', *children, ')', reference: (@temporary_reference -= 1), type: :captured_group) + end + + # If capture 1 saved the remaining input before an iteration, emit /(?!\1)/ + # to reject an iteration that leaves the input position unchanged. + def require_progress(reference) + Node.new('(?!', capture_value(reference), ')') + end + + def common_path_length(left, right) + length = 0 + length += 1 while left[length] && left[length].equal?(right[length]) + length + end + + # Paths to (a) and \1 in /(a)|\1/ => true; in /(?:(a)|\1)+/ => false, + # since different iterations can take different branches. + def mutually_exclusive?(left, right) + common_length = common_path_length(left, right) + owner = left[common_length - 1] if common_length > 0 + return false unless owner&.alternation? + + # Different branches can still participate on different loop iterations. + left[0...common_length].none? { |node| node.repeated? } + end - def null_quantify(node) - null_quantifier = Regexp::Expression::Quantifier.construct(text: '{0}') - node.update(quantifier: null_quantifier) + # Visit unbounded loops with their continuation, replacing the first suffix + # for which the caller supplies a rewrite. Do not descend into repetitions. + def rewrite_loop_and_tail(node, &block) + return false unless node.is_a?(Node) + return false if node.repeated? + + finish = sequence_end(node) + if finish + node.children.each.with_index do |child, index| + next unless child.is_a?(Node) && child.quantifier&.unbounded? && child.quantifier.minimum_at_most_one? + + following = Node.new(*node.children[(index + 1)...finish]) + replacement = yield(child, following) + next unless replacement + + node.update(children: [*node.children[0...index], replacement, *node.children[finish..-1]]) + return true + end end + node.children.any? { |child| rewrite_loop_and_tail(child, &block) } + end + + # /(a(b))/ => /(?:a(?:b))/. + def without_captures(node) + copy = node.clone + copy.walk { |part| part.make_passive if part.type == :captured_group } + copy + end + + # /a?/ => true; /a/ => false. + def may_match_empty?(node) + match_length(node).min.zero? + end + + def match_length(node) + Regexp::Parser.parse(node.to_s).match_length end end end diff --git a/lib/js_regex/second_pass/alternate_conditional_permutations.rb b/lib/js_regex/second_pass/alternate_conditional_permutations.rb new file mode 100644 index 0000000..1ae919c --- /dev/null +++ b/lib/js_regex/second_pass/alternate_conditional_permutations.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +# Replace capture conditionals with alternatives that JavaScript can match. +# Each alternative copies the pattern for one combination of participating +# groups, requires or excludes those groups, and selects the matching branches. +# Example: /(a)?(?(1)b|c)/ => /(){0}c|(a)b/ +# with redundant noncapturing groups omitted. +class JsRegex::SecondPass::AlternateConditionalPermutations < JsRegex::SecondPass + def call + permutations = conditional_tree_permutations + return if permutations.empty? + + alternatives = permutations.map.with_index do |variant, i| + Node.new((i.zero? ? '(?:' : '|(?:'), variant, ')', type: :branch) + end + tree.update(children: alternatives) + end + + private + + # For /(a)?(?(1)b|c)/, produce an absent variant and a present variant: + # /(){0}c/ and /(a)b/ (redundant noncapturing groups omitted). + def conditional_tree_permutations + conds = conditions + return [] if conds.empty? + + caps_per_branch = captured_group_count + + condition_permutations(conds).map.with_index do |truthy_conds, i| + tree_permutation = tree.clone + # find referenced groups and conditionals and make one-sided + tree_permutation.walk do |node| + build_permutation(node, conds, truthy_conds, caps_per_branch, i) + end + truthy_conds.each do |reference| + require_group_participation(tree_permutation, reference + caps_per_branch * i) + end + tree_permutation + end + end + + # /(a)?(?(1)b|c)(?(1)d|e)/ => [1], counting the repeated condition once. + def conditions + tree.walk.select { |node| node.type == :conditional }.map(&:reference).uniq + end + + # /(a)(?:b(c))/ => 2. + def captured_group_count + tree.walk.count { |node| node.type == :captured_group } + end + + # [1, 2] => [[], [1], [2], [1, 2]], the captures present in each variant. + def condition_permutations(conditions) + (0..(conditions.length)).inject([]) do |arr, n| + arr + conditions.combination(n).to_a + end + end + + # With one capture per variant, \1 in the second variant => \2. + def build_permutation(node, conds, truthy_conds, caps_per_branch, i) + truthy = truthy_conds.include?(node.reference) + + case node.type + when :backref, :capture_check + node.renumber(node.reference + caps_per_branch * i) + when :captured_group + # if the group is referenced by any condition, modulate its quantity + if conds.include?(node.reference) + adapt_referenced_group_to_permutation(node, truthy) + end + node.renumber(node.reference + caps_per_branch * i) + when :conditional + select_conditional_branch(node, truthy) + end + end + + # /(a)?/ => /(a)/ when present, or /(){0}/ when absent. + def adapt_referenced_group_to_permutation(group_node, truthy) + if truthy + min_quantify(group_node) + elsif group_node.optional? + discard_contents(group_node) + else + # Keep mandatory captures mandatory. An enclosing optional group or + # another alternative may still allow this capture not to participate. + group_node.children.insert(1, Node.new('(?!)')) + end + end +end diff --git a/lib/js_regex/second_pass/delimit_backrefs.rb b/lib/js_regex/second_pass/delimit_backrefs.rb new file mode 100644 index 0000000..bff22ac --- /dev/null +++ b/lib/js_regex/second_pass/delimit_backrefs.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +# Separate numeric backreferences from following literal digits with (?:). +# Otherwise JavaScript could read a reference followed by a digit, such as +# \1 followed by 0, as a different reference or an octal escape. +# Example: /(a)\1 0/x => /(a)\1(?:)0/. +class JsRegex::SecondPass::DelimitBackrefs < JsRegex::SecondPass + def call + delimit_backrefs(tree, '') + end + + # A /\1/ node with following = '0' becomes /\1(?:)/ and returns '\' + # as its first character; following = 'a' leaves it unchanged. + def delimit_backrefs(node, following) + quantifier_start = node.quantifier&.to_s&.[](0) + following = quantifier_start || following + if node.type == :backref && /\A[0-9]/.match(following) && + /\\[0-9]+\z/.match(node.children.last.to_s) + node << '(?:)' + end + + first = quantifier_start + node.children.reverse_each do |child| + child_start = child.is_a?(Node) ? delimit_backrefs(child, following) : child[0] + first = following = child_start if child_start + end + # Return the first character while visiting the children, rather than + # rendering each subtree again just to inspect its first character. + first unless node.dropped? + end +end diff --git a/lib/js_regex/second_pass/expand_bounded_capture_loops.rb b/lib/js_regex/second_pass/expand_bounded_capture_loops.rb new file mode 100644 index 0000000..f1b2f08 --- /dev/null +++ b/lib/js_regex/second_pass/expand_bounded_capture_loops.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +# Expand finite repetitions so each iteration has its own capture groups. +# This preserves earlier captures when later iterations skip those groups, +# and preserves Ruby's stopping rule for repeated empty iterations. +# Example: /(?:(a)|b){2}\1/ => /(?:(a)|b)(?:(a)|b)history[2,1]/. +# The history is resolved later to the latest participating copy of (a). +class JsRegex::SecondPass::ExpandBoundedCaptureLoops < JsRegex::SecondPass + def call + @references = reference_counts(tree).keys + return if @references.empty? + + expand(tree) + renumber_captures if @expanded + end + + private + + # /(?:(a)|b){2}\1/ unrolls its loop; /(a){2}\1/ stays as-is because + # every iteration necessarily updates the referenced capture. + def expand(node) + return unless node.is_a?(Node) + + node.children.each { |child| expand(child) } + quantity = node.quantifier + return unless quantity&.bounded_repetition? + + return unless node.walk.any? { |part| part.type == :captured_group && @references.include?(part.reference) } + + iteration = node.with(quantifier: nil) + nullable = may_match_empty?(iteration) + references = potentially_absent_groups(iteration) + if nullable + iteration.walk { |part| references << part.reference if part.type == :captured_group } + end + return if (references & @references).empty? + + tail = unroll(iteration, quantity, nullable) + node.update(type: :plain, reference: nil, quantifier: nil, children: [tail]) + @expanded = true + end + + # Body /(?:(a)|b)/ with {2} => /(?:(a)|b)(?:(a)|b)/, before renumbering. + def unroll(iteration, quantity, nullable) + positions = {} + quantity.iteration_indices.each do |index| + positions[index - 1] = temporary_capture('[\\s\\S]*') if nullable && quantity.before_last_iteration?(index + 1) + end + tail = Node.new + quantity.iteration_indices.reverse_each do |index| + current = Node.new(iteration.clone, tail) + probe = positions[index - 1] + current.children.unshift(Node.new('(?=', probe, ')')) if probe + previous_position = index > 2 ? positions[index - 3] : nil + tail = iteration_choice(current, index, quantity, previous_position) + end + tail + end + + # For the optional second iteration of {1,2}, /a/ => /(?:a|)/; + # for {1,2}?, /a/ => /(?:|a)/. + def iteration_choice(current, index, quantity, previous_position) + # Ruby permits one empty iteration before its loop check sees the same + # position again. Keep that last empty capture when the loop stops. + if previous_position + current = Node.new(require_progress(previous_position.reference), current) + end + if quantity.optional_iteration?(index) + branches = [current, Node.new] + branches.reverse! if quantity.reluctant? + alternate(branches) + elsif previous_position + stopped = Node.new('(?=', capture_value(previous_position.reference), ')') + alternate([current, stopped]) + else + current + end + end +end diff --git a/lib/js_regex/second_pass/expand_constant_capture_loops.rb b/lib/js_regex/second_pass/expand_constant_capture_loops.rb new file mode 100644 index 0000000..afa08e8 --- /dev/null +++ b/lib/js_regex/second_pass/expand_constant_capture_loops.rb @@ -0,0 +1,287 @@ +# frozen_string_literal: true + +# Rewrite unbounded loops whose backreferences can be replaced by fixed strings. +# Track which captures have participated, then substitute their known values +# in backreferences and select conditional branches in the original choice order. +# Captures used only by conditions may vary: only their participation matters. +# Example: /(?:(a)|\1)+/ => /(a)(?:(a)|a)*/ +# with redundant noncapturing groups omitted: once set, \1 is always a. +class JsRegex::SecondPass::ExpandConstantCaptureLoops < JsRegex::SecondPass + def call + loop do + @counts = reference_counts(tree) + break if @counts.empty? || !expand(tree) + + renumber_captures + end + end + + private + + # In /x(?:(a)|\1)+y/, compile the loop together with its tail y, then + # replace that suffix; return true after one such replacement. + def expand(node) + rewrite_loop_and_tail(node) do |child, following| + catch(:unsupported_capture_loop) { compile(child, following) } + end + end + + # For /(?:(a)|\1)+/, state 0 has no capture and must choose (a) first; + # state 1 can then repeat either (a) or the known value a. + def compile(node, following) + groups = node.walk.select { |part| part.type == :captured_group && @counts[part.reference] > 0 } + return if groups.empty? + + references = groups.map(&:reference) + internal = reference_counts(node) + return unless groups.length > 1 || references.any? { |reference| internal[reference] > 0 } || + (node.quantifier.reluctant? && conditional_reference?(following, references)) + + following_counts = reference_counts(following) + return unless references.all? do |reference| + internal[reference] + following_counts[reference] == @counts[reference] + end + + @values = constant_capture_values(groups) + # One bit per capture records whether it has participated in the loop. + @capture_bits = references.each.with_index.to_h { |ref, idx| [ref, 1 << idx] } + @all_captures = (1 << references.length) - 1 + @iteration = node.with(quantifier: nil) + @following = following + @reluctant = node.quantifier.reluctant? + @expanded_states = {} + if node.optional? + remaining_iterations(0) + else + alternate(iteration_paths(0).map { |part, state| Node.new(part, remaining_iterations(state)) }) + end + end + + # In /(?:(a)|ab)+?(?(1)c|a)/, the trailing condition needs capture 1's state + # even after a later ab iteration skips it. + def conditional_reference?(node, references) + node.walk.any? do |part| + part.conditional? && + part.condition_references.any? { |ref| references.include?(ref) } + end + end + + # Capture 1 containing /(ab)/ => {1 => /(?:ab)/}; /(a+)/ is allowed only + # when conditions need its participation but no reference reads its value. + def constant_capture_values(groups) + groups.each_with_object({}) do |group, values| + expression = Regexp::Parser.parse(group.children.join) + unless constant_word(expression) + throw :unsupported_capture_loop if capture_value_referenced?(group.reference) + next + end + + values[group.reference] = without_captures(group).update(quantifier: nil) + end + end + + # /(a+)(?(1)b|c)/ => false for capture 1; /(a+)\1/ => true. + def capture_value_referenced?(reference) + tree.walk.any? do |node| + [:backref, :capture_check].include?(node.type) && node.reference == reference + end + end + + # Once capture 1 is set to a, the remaining /(?:(a)|\1)*/ iterations can + # use /(?:(a)|a)*/ (redundant noncapturing groups omitted). + def remaining_iterations(state) + return @expanded_states[state].clone if @expanded_states.key?(state) + + if state == @all_captures + body = substitute_known_captures(@iteration, state) + tail = substitute_known_captures(@following, state) + result = Node.new(repeat(body, @reluctant), tail) + else + result = repeat_without_new_captures(continuation_options(state)) + end + @expanded_states[state] = result + end + + # Each option either repeats the current state or continues to a new one. + # Their order determines which choices backtracking tries first. + # For /(?:(a)|b)*/ in state 0, (a) advances to state 1, b repeats state 0, + # and stopping comes last; for *?, stopping comes first. + def continuation_options(state) + options = iteration_paths(state).map do |part, next_state| + next_state == state ? [part, nil] : [nil, Node.new(part, remaining_iterations(next_state))] + end + ending = [nil, substitute_known_captures(@following, state)] + @reluctant ? options.unshift(ending) : options.push(ending) + options.reject! { |_, continuation| continuation && unmatchable?(continuation) } + options + end + + # Options [[/b/, nil], [nil, /c/]] => /(?:b)*c/: repeat b, then continue at c. + def repeat_without_new_captures(options) + loops = options.map(&:first).compact + return alternate(options.map(&:last)) if loops.empty? + + first_loop = options.index { |part, _| part } + last_loop = options.rindex { |part, _| part } + before = options[0...first_loop].map(&:last) + after = options[(last_loop + 1)..-1].map(&:last) + throw :unsupported_capture_loop if options[first_loop..last_loop].any? { |_, continuation| continuation } + + # Regrouping these iterations must leave only one way to split the text. + # Otherwise the regrouping could change backtracking order. + unless before.empty? || after.empty? || unique_iteration_boundaries?(loops) + throw :unsupported_capture_loop + end + loop_body = alternate(loops) + branches = [] + branches << Node.new(repeat(loop_body.clone, true), alternate(before)) unless before.empty? + branches << Node.new(repeat(loop_body, false), alternate(after)) unless after.empty? + alternate(branches) + end + + # [/a/, /b/] => true; [/a/, /ab/] => false because one word prefixes another. + def unique_iteration_boundaries?(loops) + words = loops.map { |part| constant_word(Regexp::Parser.parse(part.to_s)) } + return false if words.any? { |word| !word || word.empty? } + + words.map!(&:downcase) + words.combination(2).none? do |left, right| + left.start_with?(right) || right.start_with?(left) + end + end + + # For /(?:(a)|\1)/ in state 0, keep only [(a), state 1]; \1 cannot yet match. + def iteration_paths(state) + expand_iteration(@iteration, state).reject { |part, _| unmatchable?(part) }.each do |part, _| + throw :unsupported_capture_loop if may_match_empty?(part) + end + end + + # With bit 1 assigned to capture 1, /(?:(a)|b)/ in state 0 yields + # [(a), state 1] and [b, state 0]. + def expand_iteration(node, state) + return [[node.clone, state]] if node.is_a?(String) + + updates = node.walk.any? do |part| + part.type == :captured_group && @capture_bits.key?(part.reference) + end + return [[substitute_known_captures(node, state), state]] unless updates + + return optional_iteration_paths(node, state) if node.quantifier + if node.conditional? + selected = conditional_branch(node, state) + throw :unsupported_capture_loop unless selected + + return expand_iteration(selected, state) + end + head = node.children.first + throw :unsupported_capture_loop if head.is_a?(String) && head.match?(/\A\((?!\?:\z|\z)/) + + if node.alternation? + return node.children.flat_map do |branch| + child = branch.with(type: :plain) + child.children.shift if child.children.first == '|' + expand_iteration(child, state) + end + end + sequence_paths(node, state) + end + + # /(a)?/ in state 0 => [(a), state 1], ['', state 0]; ?? reverses the order. + def optional_iteration_paths(node, state) + throw :unsupported_capture_loop unless node.zero_or_one? + + present = expand_iteration(node.with(quantifier: nil), state) + absent = [[Node.new, state]] + node.quantifier.reluctant? ? absent + present : present + absent + end + + # /(a)\1/ starting in state 0 => /(a)(?:(?:a))/ in state 1: + # the first child sets the capture before the second child reads it. + def sequence_paths(node, state) + results = [[[], state]] + node.children.each do |child| + results = results.flat_map do |parts, previous_state| + following = expand_iteration(child, previous_state) + if following.length > 1 && parts.any? { |part| can_backtrack?(part) } + throw :unsupported_capture_loop + end + following.map { |part, next_state| [[*parts.map(&:clone), part], next_state] } + end + end + results.map do |parts, next_state| + next_state |= @capture_bits[node.reference] if node.type == :captured_group && @capture_bits.key?(node.reference) + [node.with(children: parts), next_state] + end + end + + # If capture 1 holds a, \1 => /(?:(?:a))/; if it has not participated, + # \1 => /(?:(?!))/ instead. + def substitute_known_captures(node, state) + return node.clone if node.is_a?(String) + + case node.type + when :backref, :capture_backref, :multiplex_backref + references = node.type == :backref ? [node.reference] : node.children.map(&:reference) + if references.any? { |reference| @capture_bits.key?(reference) } + throw :unsupported_capture_loop unless references.all? { |reference| @capture_bits.key?(reference) } + + reference = references.find { |number| state & @capture_bits[number] != 0 } + value = reference ? @values[reference].clone : Node.new('(?!)') + return node.with(children: ['(?:', value, ')'], type: :plain, reference: nil) + end + when :conditional, :capture_conditional + selected = conditional_branch(node, state) + if selected + return node.with(children: ['(?:', substitute_known_captures(selected, state), ')'], type: :plain, reference: nil) + end + end + node.with(children: node.children.map { |child| substitute_known_captures(child, state) }) + end + + # (?(1)b|c) selects b when bit 1 is set, and c when it is clear. + def conditional_branch(node, state) + references = node.condition_references + return unless references.any? { |reference| @capture_bits.key?(reference) } + + throw :unsupported_capture_loop unless references.all? { |ref| @capture_bits.key?(ref) } + present = references.any? { |reference| state & @capture_bits[reference] != 0 } + branches = node.conditional_branches + (present ? branches[0] : branches[1]) || Node.new + end + + # /(?:ab){2}/ => 'abab'; /a?/ => nil because its value can vary. + def constant_word(expression) + quantifier = Node::Quantifier.from_expression(expression) + return unless !quantifier || quantifier.fixed? + return '' if quantifier&.zero? + + word = + case expression.type + when :literal then expression.text + when :group, :expression + children = expression.expressions.map { |child| constant_word(child) } + children.join if children.all? + when :escape + characters = CharacterSet.of_expression(expression) + characters.first.chr(Encoding::UTF_8) if characters.size == 1 + end + + (quantifier ? quantifier.repeat_word(word) : word) if word + end + + # /a+/ => true; /a{2}/ => false. + def can_backtrack?(node) + return false if node.is_a?(String) + return true if node.quantifier && !node.quantifier.fixed? + return true if node.internal_reference? + return true if node.children.any? { |child| (child.is_a?(Node) && child.branch?) } + + node.children.any? { |child| can_backtrack?(child) } + end + + # (/a/, false) => /(?:a)*/; (/a/, true) => /(?:a)*?/. + def repeat(node, reluctant) + Node.new('(?:', node, ')').quantify(0, -1, reluctant: reluctant) + end +end diff --git a/lib/js_regex/second_pass/expand_nullable_captures.rb b/lib/js_regex/second_pass/expand_nullable_captures.rb new file mode 100644 index 0000000..28b783e --- /dev/null +++ b/lib/js_regex/second_pass/expand_nullable_captures.rb @@ -0,0 +1,327 @@ +# frozen_string_literal: true + +# Separate choices that skip a capture from choices that capture an empty string. +# Copy the rest of the pattern into each choice so references and conditionals +# use that choice's capture state without changing greedy or lazy match order. +# For nullable loops, keep the final empty capture outside the repeated body. +# Example: /(a*)?\1/ => /(?:(a*)\1|(a*){0}(?!))/. +# The absent choice fails; the present choice may still capture an empty string. +class JsRegex::SecondPass::ExpandNullableCaptures < JsRegex::SecondPass + def call + return unless tree.walk.any? do |node| + node.type == :captured_group && !node.marked_nonempty? + end + + loop do + @counts = reference_counts(tree) + break if @counts.empty? || !expand(tree) + + renumber_captures + end + end + + private + + # /(a)(?:b(c))/ => the capture nodes numbered [1, 2]. + def captures(node) + node.walk.select { |part| part.type == :captured_group } + end + + # In /b(a*)?\1c/, replace /(a*)?\1c/ together, so each choice gets its + # own continuation; return true after that one replacement. + def expand(node, repeated = false) + return false unless node.is_a?(Node) + + repeated ||= node.repeated? + return true if node.children.any? { |child| expand(child, repeated) } + return false if repeated + + # Compound atomic wrappers cannot be treated as ordinary sequences. + finish = sequence_end(node) + return false unless finish + + node.children.each.with_index do |child, index| + next unless child.is_a?(Node) + + groups = captures(child) + candidates = groups.reject do |capture| + capture.marked_nonempty? || @counts[capture.reference].zero? + end + next if candidates.empty? + + following = Node.new(*node.children[(index + 1)...finish]) + suffix_counts = reference_counts(following) + next unless candidates.any? do |capture| + suffix_counts[capture.reference] == @counts[capture.reference] + end + + replacement = expand_choice(child, following, groups) + next unless replacement + + node.update(children: [*node.children[0...index], replacement, *node.children[finish..-1]]) + return true + end + false + end + + # For /(?:(a*)?)\1/, move the continuation into the passive group: + # /(?:(a*)?\1)/, allowing a subsequent expansion of the inner optional capture. + def expand_choice(child, following, groups) + if groups.length == 1 + if greedy_nullable_loop?(child) + return expand_nullable_loop(child, following) + elsif child.quantifier&.reluctant? + collapsed = collapse_lazy_character_run(child, following) + return collapsed if collapsed + end + end + + if child.quantifier&.unbounded? && child.quantifier.minimum_at_most_one? && !child.quantifier.reluctant? + replacement = expand_nullable_alternation(child, following, groups) + return replacement if replacement + end + + choices = choices_for(child) + if choices + original_groups = groups.map(&:reference) + alternate(choices.map do |choice| + continuation = following.clone + specialize(continuation, capture_states(choice, original_groups)) + Node.new(choice, continuation, type: :branch) + end) + elsif passive_sequence?(child) + position = child.children.first == '(?:' ? child.children.length - 1 : child.children.length + child.children.insert(position, *following.children) + child + end + end + + # /(a*)+?\1/ => /a*()/: the greedy run first consumes every a. Its reference + # then fails unless the run was empty, so Ruby takes one final empty iteration. + # Restrict this to a single-character run and a lone required reference; + # a longer continuation can force backtracking to a different capture value. + def collapse_lazy_character_run(capture, following) + quantity = capture.quantifier + return unless capture.type == :captured_group && quantity&.reluctant? && + quantity.unbounded? && quantity.minimum_at_most_one? + + reference = following.children.first + return unless following.children.one? && reference.is_a?(Node) && reference.type == :backref && + reference.reference == capture.reference && !reference.quantifier + + body = Node.new(*capture.children[1...-1]) + expression = single_expression(Regexp::Parser.parse(body.to_s)) + repeat = Node::Quantifier.from_expression(expression) + return unless repeat&.zero_or_more? && !repeat.reluctant? + + expression.quantifier = nil + expression = single_expression(expression) + return unless CODEPOINT_TYPES[expression.type] && + (ml = expression.match_length).fixed? && ml.min == 1 + + Node.new(body, empty_capture(capture)) + end + + CODEPOINT_TYPES = %i[literal set type property nonproperty escape].to_h { |k| [k, true] }.freeze + + # /(?:a)/ => the a expression; /(?:ab)*/ stays wrapped because it is quantified. + def single_expression(expression) + while !expression.quantifier && + [:group, :expression].include?(expression.type) && + expression.expressions.one? + expression = expression.expressions.first + end + expression + end + + # A nullable alternative can end the loop before a later, consuming + # alternative is tried. Test that stopping choice before each nonempty step. + # For /(?:(a*)|b)+\1/ on 'b', try the final empty capture before consuming b. + # Shape: guarded nonempty iterations, then /()\1/ or a retained-capture tail. + def expand_nullable_alternation(loop_node, following, groups) + referenced = groups.select { |group| @counts[group.reference] > 0 } + return unless referenced.length == 1 && referenced.first.nullable? + return unless reference_counts(loop_node).empty? + + capture = referenced.first + iteration = loop_node.with(quantifier: nil) + choices = split_alternatives(iteration) + return unless choices && choices.length > 1 + + empty_choices = choices.select { |choice| captures(choice).any? { |group| group.reference == capture.reference } } + return unless empty_choices.length == 1 && capture_only?(empty_choices.first, capture.reference) + return if (choices - empty_choices).all? { |choice| may_match_empty?(choice) } + + present_tail = following.clone + specialize(present_tail, capture.reference => true) + probe_body = Node.new('(', iteration.clone, ')', reference: capture.reference) + stop = stop_before_empty_capture(probe_body, present_tail) + position = temporary_capture('[\\s\\S]*') + mark_nonempty_capture(iteration, capture.reference) + prefix = Node.new('(?:', stop, Node.new('(?=', position, ')'), iteration, + require_progress(position.reference), ')') + prefix.quantify(0, -1) + + empty = empty_choices.first + empty.walk do |part| + part.update(children: ['(', ')'], quantifier: nil) if part.type == :captured_group && part.reference == capture.reference + end + retained_tail = following.clone + retained_tail.walk do |part| + make_condition_dynamic(part) if part.conditional? && part.reference == capture.reference + end + unless loop_node.optional? + start = temporary_capture('[\\s\\S]*') + prefix = Node.new(Node.new('(?=', start, ')'), prefix) + retained_tail = Node.new(require_progress(start.reference), retained_tail) + end + Node.new(prefix, alternate([Node.new(empty, present_tail), retained_tail])) + end + + # For capture 1, /(?:(a*))/ => true; /(?:b(a*))/ => false. + def capture_only?(node, reference) + return ['(', '(?:', ')', ''].include?(node) if node.is_a?(String) + return !node.quantifier if node.type == :captured_group && node.reference == reference + return false if node.quantifier + + node.children.all? { |child| capture_only?(child, reference) } + end + + # Inside a guarded consuming iteration, (a*) keeps its source but changes + # its nonempty annotation from false to true. + def mark_nonempty_capture(node, reference) + node.children.map! do |child| + next child if child.is_a?(String) + + if child.type == :captured_group && child.reference == reference + child.with(children: child.children, marked_nonempty: true, nullable: false) + else + mark_nonempty_capture(child, reference) + child + end + end + end + + # /(a*)+/ => true; /(a*)+?/ => false. + def greedy_nullable_loop?(node) + node.type == :captured_group && node.nullable? && !!node.quantifier&.unbounded? && node.quantifier.minimum_at_most_one? && !node.quantifier.reluctant? + end + + # /(?:a(b))/ => true; /(?:a(b))*/ => false because it is repeated. + def passive_sequence?(node) + return false unless node.type == :plain && !node.quantifier && node.children.any? { |child| child.is_a?(Node) } + + node.children.first == '(?:' || !node.children.any? { |child| child.is_a?(String) } + end + + # /(a*)?/ => [/(a*)/, /(a*){0}/]; /(a*)??/ gives the reverse order. + def choices_for(node) + if node.zero_or_one? + present = node.with(quantifier: nil) + absent = node.clone + discard_contents(absent) + node.quantifier.reluctant? ? [absent, present] : [present, absent] + elsif !node.quantifier + split_alternatives(node) + end + end + + # Shape: /(a*)+\1/ => guarded nonempty iterations followed by either + # /()\1/ or /\1/ using the last nonempty capture. The empty ending is tried first. + def expand_nullable_loop(capture, following) + present_tail = following.clone + specialize(present_tail, capture.reference => true) + prefix = nonempty_iterations(capture, present_tail) + ending = alternate([ + Node.new(empty_capture(capture), present_tail), + last_nonempty_tail(capture, following), + ]) + Node.new(prefix, ending) + end + + # For the repeated body (a*), consuming 'aaa' is allowed; consuming '' is + # blocked by a position check and left to the separate empty ending. + def nonempty_iterations(capture, present_tail) + stop = stop_before_empty_capture(capture, present_tail) + remainder = temporary_capture('[\\s\\S]*') + participating = capture.with(quantifier: nil, marked_nonempty: true, nullable: false) + prefix = Node.new('(?:', stop, Node.new('(?=', remainder, ')'), participating, + require_progress(remainder.reference), ')') + prefix.quantify(0, -1) + end + + # With capture /(a*)/ and tail /\1b/, input 'b' selects the empty ending; + # the emitted negative lookahead prevents another loop iteration there. + def stop_before_empty_capture(capture, following) + # Prefer Ruby's final empty iteration when the body first chooses empty + # and the rest of the pattern succeeds with that value. + probe = temporary_capture(*capture.children[1...-1].map(&:clone)) + tail = following.clone + tail.walk do |part| + next unless part.reference == capture.reference && part.type != :captured_group + + part.renumber(probe.reference) + end + Node.new('(?!', Node.new('(?=', probe, ')'), nonempty_capture_check(probe.reference, false), tail, ')') + end + + # For /(a*)+/ followed by /\1/, emit /(?=[\s\S]*(?![\s\S])(?!\1))\1/, + # so this ending requires a previous nonempty capture. + def last_nonempty_tail(capture, following) + tail = following.clone + tail.walk do |part| + make_condition_dynamic(part) if part.conditional? && part.reference == capture.reference + end + capture.optional? ? tail : Node.new(nonempty_capture_check(capture.reference, true), tail) + end + + # Choice /(a*)/ from /(?:(a*)|(b*))/ => {1 => true, 2 => false}. + def capture_states(node, original_groups) + found = captures(node).map(&:reference) + states = (original_groups - found).to_h { |ref| [ref, false] } + collect_states(node, states) + states + end + + # /(a*)(b*)?(c*){0}/ => {1 => true, 3 => false}; capture 2 stays unknown. + def collect_states(node, states, required = true, excluded = false) + return unless node.is_a?(Node) + + excluded ||= node.zero_quantified? + required &&= !node.optional? && !node.negative_assertion? + if node.type == :captured_group + states[node.reference] = false if excluded + states[node.reference] = true if required && !excluded + end + required &&= !node.conditional? && + !node.alternation? + node.children.each { |child| collect_states(child, states, required, excluded) } + end + + # With {1 => false}, \1 => /(?!)/ and (?(1)a|b) selects b. + # With {1 => true}, the same condition selects a, even for an empty capture. + def specialize(node, states) + return unless node.is_a?(Node) + + case node.type + when :backref + node.make_unmatchable if states[node.reference] == false + when :multiplex_backref, :capture_backref + children = node.children.reject { |child| states[child.reference] == false } + if children.empty? + node.make_unmatchable + else + node.update(children: children) + end + when :conditional + select_conditional_branch(node, states[node.reference]) if states.key?(node.reference) + when :capture_conditional + references = node.condition_references + present = references.any? { |reference| states[reference] } + if present || references.all? { |reference| states[reference] == false } + select_conditional_branch(node, present) + end + end + node.children.each { |child| specialize(child, states) } + end +end diff --git a/lib/js_regex/second_pass/handle_non_participating_backrefs.rb b/lib/js_regex/second_pass/handle_non_participating_backrefs.rb new file mode 100644 index 0000000..2ee082b --- /dev/null +++ b/lib/js_regex/second_pass/handle_non_participating_backrefs.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +# Make backreferences to skipped groups fail, as in Ruby, instead of matching +# an empty string as in JavaScript. Check capture values when they cannot be +# empty; otherwise, require the group to participate where the reference is required. +# Example: /\1(a)/ => /(?!)(a)/: the forward reference cannot succeed. +class JsRegex::SecondPass::HandleNonParticipatingBackrefs < JsRegex::SecondPass + def call + nodes, paths = reference_nodes_with_paths + groups = {} + nodes.each do |node| + if node.type == :captured_group + groups[node.reference] = node + elsif node.type == :backref + guard_backreference(node, groups[node.reference], paths) + end + end + end + + private + + # /(a)|b\1/ => /(a)|b(?!)/: capture and reference are in different branches. + def guard_backreference(node, capture, paths) + path = paths[node] + capture_path = capture && paths[capture] + if !capture || path.include?(capture) || mutually_exclusive?(capture_path, path) + node.make_unmatchable + return + end + + skipped = capture_path.select { |ancestor| skippable_scope?(ancestor) } - path + if skipped.any? { |ancestor| ancestor.negative_assertion? } + node.make_unmatchable + elsif skipped.any? + require_participation(node, capture, capture_path, path) + end + end + + # /(a)?\1/ => /(a)?(?:(?=[\s\S]*(?![\s\S])(?!\1))\1)/, + # checking the capture at the reference without forcing the optional group. + def require_participation(node, capture, capture_path, path) + if capture.marked_nonempty? + reference = node.with(quantifier: nil) + node.update(type: :plain, reference: nil, children: ['(?:', nonempty_capture_check(reference.reference, true), reference, ')']) + else + require_nullable_capture(node, capture_path, path) + end + end + + # /(?:(a*)?\1)?/ => /(?:(a*)\1)?/: require the capture inside the shared + # scope, while keeping that entire scope optional. + def require_nullable_capture(backref, group_path, path) + return if path[-2].walk.any? do |node| + node.type == :capture_check && node.reference == backref.reference + end + + common_length = common_path_length(group_path, path) + common = group_path[common_length - 1] + remaining_path = path[common_length..-1] + return if remaining_path.any? { |ancestor| skippable_scope?(ancestor) } + + # Every match through this node uses the reference, so paths that skip its + # capture cannot succeed. Keep the enclosing node's own optionality intact. + quantifier = common.quantifier + common.update(quantifier: nil) + require_group_participation(common, backref.reference) + common.update(quantifier: quantifier) + end +end diff --git a/lib/js_regex/second_pass/preserve_repeated_captures.rb b/lib/js_regex/second_pass/preserve_repeated_captures.rb new file mode 100644 index 0000000..cef07db --- /dev/null +++ b/lib/js_regex/second_pass/preserve_repeated_captures.rb @@ -0,0 +1,370 @@ +# frozen_string_literal: true + +# Rewrite unbounded repetitions so captures survive later iterations that skip +# their groups, as in Ruby. Isolate the last capturing iteration, or group each +# capturing iteration with the intervening iterations that skip the group. +# Example shape: /(?:(a)|b)+\1/ => /(?:b*?(a))+b*\1/. +# This shows the successful capturing path, omitting disabled branches and the +# separate path with no capture, whose required backreference will fail. +class JsRegex::SecondPass::PreserveRepeatedCaptures < JsRegex::SecondPass + def call + collect_references + return if @references.empty? + + while @references.uniq.length > 1 && expand_lazy_capture_loops(tree) + renumber_captures + collect_references + end + rewrite_repetitions(tree) + renumber_captures if @rewritten + end + + private + + # /(a)\1/ => references [1], mandatory references [1]; /(a)\1?/ keeps + # reference 1 but does not mark it mandatory. + def collect_references + @references = [] + @groups = {} + group_paths = {} + reference_paths = [] + tree.walk(trace: true) do |node, event, path| + next if event == :exit + + if node.type == :captured_group + @groups[node.reference] = node + group_paths[node.reference] = path.dup + elsif node.reference + @references << node.reference + end + if node.type == :backref && path.none? { |ancestor| [:multiplex_backref, :capture_backref].include?(ancestor.type) } + reference_paths << [node.reference, path.dup] + end + end + # A reference can be required within its own branch or optional parent, + # even when that entire scope can be skipped by the rest of the pattern. + @mandatory_references = reference_paths.select do |reference, reference_path| + required_on_path?(reference_path, group_paths[reference] || []) + end.map(&:first) + end + + # For /(?:(a)|b)+\1/, choose capture-update cycles; for {2}, leave the loop + # to ExpandBoundedCaptureLoops. + def rewrite_repetitions(node) + return unless node.is_a?(Node) + + node.children.each { |child| rewrite_repetitions(child) } + quantifier = node.quantifier + return unless quantifier&.unbounded? + + iteration = node.with(quantifier: nil) + plan = nil + reference = potentially_absent_groups(iteration, reachable_only: true).find do |number| + @references.include?(number) && (@groups[number].marked_nonempty? || + @groups[number].nullable?) && + (plan = rewrite_plan(iteration, number, quantifier)) + end + return unless reference + + kind, capture_first = plan + if kind == :last + isolate_last_capture(node, iteration, reference, quantifier) + else + repeat_capture_updates(node, iteration, reference, quantifier, + capture_first: capture_first, internal: kind == :retained) + end + @rewritten = true + rewrite_repetitions(node) + end + + # In /x(?:(a+)|(b+))+?\1\2y/, replace the loop together with /\1\2y/, + # so its ending can test the continuation before choosing iteration boundaries. + def expand_lazy_capture_loops(node) + rewrite_loop_and_tail(node) do |child, following| + lazy_capture_loop(child, following) if child.quantifier.reluctant? + + + end + end + + # Shape for /(?:(a+)|(b+))+?\1\2/: optional b-run, repeated a/b run pairs, + # then guarded endings. Each run retains its last capture for the continuation. + def lazy_capture_loop(node, following) + alternatives = alternating_capture_runs(node, following) + return unless alternatives + + left, right = alternatives.map { |part| Node.new(lazy_run(without_captures(part), 0), part) } + pair = Node.new(left.clone, right.clone) + ending = lazy_loop_endings(alternatives.first, left, right, pair, following) + prefix = Node.new('(?:', right, ')').quantify(0, -1, reluctant: true) + Node.new(prefix, lazy_run(pair, 0), ending) + end + + # /(?:(a+)|(b+))+?\2\1/ => [/(b+)/, /(a+)/], ordered by the following refs. + def alternating_capture_runs(node, following) + return unless reference_counts(node).empty? + + iteration = node.with(quantifier: nil) + alternatives = split_alternatives(iteration) + return unless alternatives&.length == 2 + + references = alternatives.map { |alternative| referenced_run(alternative) } + return unless references.all? && references.uniq.length == 2 + + first, second = following.children + return unless [first, second].all? { |part| part.is_a?(Node) && part.type == :backref && !part.quantifier } + return unless [first.reference, second.reference].sort == references.sort + + counts = reference_counts(following) + return unless references.all? { |reference| counts[reference] == 1 && @references.count(reference) == 1 } + + alternatives.reverse! if first.reference != references.first + expressions = alternatives.map { |part| Regexp::Parser.parse(part.to_s) } + alternatives if disjoint_starts?(*expressions) + end + + # An alternative /(a+)/ with capture 1 referenced => 1; /(ab+)/ => nil, + # since it is not a run of a single character. + def referenced_run(alternative) + groups = alternative.walk.select do |part| + part.type == :captured_group && @references.include?(part.reference) + end + return unless groups.length == 1 && groups.first.marked_nonempty? + return if potentially_absent_groups(alternative).include?(groups.first.reference) + + groups.first.reference if capture_run?(alternative, groups.first) + end + + # For /(?:(a+)|(b+))+?\1\2/, endings try the tail, one final a capture, + # then a split a-run; probes prevent these from outranking an earlier iteration. + def lazy_loop_endings(first, left, right, pair, following) + continuation = Node.new(right.clone, lazy_run(pair.clone, 0), alternate([ + following.clone, Node.new(left.clone, following.clone), + ])) + raw_future = Node.new(first.clone, continuation.clone) + discarded = without_captures(first) + + # A final first-character run has two stopping choices: shorten its first + # iteration, or make another iteration. Prefer any successful continuation + # from an earlier (longer) first iteration before accepting either choice. + single_probe = Node.new('(?=', first.clone, following.clone, ')') + shorter = Node.new(discarded.clone, Node.new('(?!', capture_value(following.children.first.reference), ')'), + first.clone, continuation.clone) + single_tail = Node.new(single_probe, Node.new('(?!', alternate([raw_future.clone, shorter]), ')'), + first.clone, following.clone) + + discarded_prefix = temporary_capture(discarded.clone) + split_probe = Node.new('(?=', discarded_prefix, first.clone, following.clone, ')') + earlier = Node.new(capture_value(discarded_prefix.reference), lazy_run(discarded, 0), + first.clone, continuation) + split_tail = Node.new(split_probe, Node.new('(?!', alternate([raw_future, earlier]), ')'), + backref(discarded_prefix.reference), first.clone, following.clone) + alternate([following.clone, single_tail, split_tail]) + end + + # Restrict this rewrite to greedy runs of one character, with the capture + # covering the entire alternative. Splitting more general words can change + # the order in which a loop explores its iteration boundaries. + # /(a+)/ => true; /(a+?)/ => false; /(ab+)/ => nil (also ineligible). + def capture_run?(node, capture) + return ['(', '(?:', ')', ''].include?(node) if node.is_a?(String) + if node.equal?(capture) + return false if node.quantifier || node.walk.any? { |part| part.quantifier&.reluctant? } + + expression = Regexp::Parser.parse(node.children.join) + while !expression.terminal? && !expression.quantifier && expression.expressions.length == 1 + expression = expression.expressions.first + end + quantifier = Node::Quantifier.from_expression(expression) + return !!quantifier&.one_or_more? && + !quantifier.reluctant? && expression.match_length.min == 1 && + CharacterSet.of_expression(expression).size == 1 + end + return false if node.quantifier + + node.children.all? { |child| capture_run?(child, capture) } + end + + # (/a/, 0) => /(?:a){0,}?/; (/a/, 1) => /(?:a){1,}?/. + def lazy_run(node, minimum) + Node.new('(?:', node, ')').quantify(minimum, -1, reluctant: true) + end + + # Capturing-path shape: /(?:(a)|b)+/ => /(?:a|b)*(a)b*/. + # Earlier (a) copies become passive; the final copy survives all later b's. + def isolate_last_capture(node, iteration, reference, quantifier) + absent = iteration.clone + remove_capture(absent, reference, exclude: true) + absent.update(quantifier: quantifier) + branches = quantifier.prefix_counts.map do |prefix_min, prefix_max| + prefix = iteration.clone + remove_capture(prefix, reference, exclude: false) + prefix.quantify(prefix_min, prefix_max, reluctant: quantifier.reluctant?) + last_update = iteration.clone + require_group_participation(last_update, reference) + suffix = iteration.clone + remove_capture(suffix, reference, exclude: true) + suffix_min = quantifier.remaining_minimum(prefix_min + 1) + suffix.quantify(suffix_min, -1, reluctant: quantifier.reluctant?) + Node.new(prefix, last_update, suffix) + end + replace_loop(node, branches, absent, quantifier.reluctant?) + end + + # For the loop in /(?:(a)|b)+\1/, return [:cycles, true]: each cycle updates + # capture 1, and its alternative is tried before the skipping alternative. + def rewrite_plan(iteration, reference, quantifier) + target = nil + internal_reference = false + other_capture = false + iteration.walk do |node| + target = node if node.type == :captured_group && node.reference == reference + other_capture ||= node.type == :captured_group && node.reference != reference && @references.include?(node.reference) + internal_reference ||= node.internal_reference? + end + capturing = iteration.clone + require_group_participation(capturing, reference) + skipping = iteration.clone + remove_capture(skipping, reference, exclude: true) + captured_expression = Regexp::Parser.parse(analysis_source(capturing)) + skipped_expression = Regexp::Parser.parse(analysis_source(skipping)) + mandatory = @mandatory_references.include?(reference) + single_capture = (potentially_absent_groups(iteration, reachable_only: true) & @references).uniq.length == 1 + if !internal_reference && !other_capture && single_capture && target.marked_nonempty? && quantifier.minimum_at_most_one? && + captured_expression.match_length.min > 0 && skipped_expression.match_length.min > 0 + first = capture_at_edge?(iteration, reference, first: true) + last = capture_at_edge?(iteration, reference, first: false) + if (first && !quantifier.reluctant?) || (mandatory && (first || last)) + return [:cycles, first] + end + end + + disjoint = disjoint_starts?(captured_expression, skipped_expression) + capture = Regexp::Parser.parse(target.children.join) + empty_capture_safe = target.nullable? && disjoint && !quantifier.reluctant? + single_character_run = target.marked_nonempty? && CharacterSet.of_expression(capture).size == 1 && !quantifier.reluctant? + if !internal_reference && (capture.match_length.fixed? || empty_capture_safe || single_character_run) && + (disjoint || (single_capture && mandatory && !quantifier.reluctant?)) + [:last, nil] + elsif (disjoint || (single_capture && mandatory && quantifier.reluctant?)) && + quantifier.minimum_at_most_one? && skipped_expression.match_length.min > 0 + [:retained, nil] + end + end + + # /a+/ and /b+/ => true; /a/ and /A/ => false after case folding. + def disjoint_starts?(left, right) + left_characters = first_characters(left).case_insensitive + right_characters = first_characters(right).case_insensitive + left_characters.disjoint?(right_characters) + end + + # Whether branch and optional choices try the capture first or last. + # In /(?:(a)|b)/, capture 1 is first: true, last: false; + # /(?:(a)??)/ is first: false, last: true because skipping is tried first. + def capture_at_edge?(node, reference, first:) + return false if node.is_a?(String) + return false if node.optional? && node.quantifier.reluctant? == first + return true if node.type == :captured_group && node.reference == reference + + if node.alternation? + capture_at_edge?(first ? node.children.first : node.children.last, reference, first: first) + else + node.children.any? { |child| capture_at_edge?(child, reference, first: first) } + end + end + + # Capturing-path shape: /(?:(a)|b)+/ => /(?:b*?(a))+b*/. + # Every repeated cycle contains (a), so trailing b iterations cannot clear it. + def repeat_capture_updates(node, iteration, reference, quantifier, capture_first:, internal:) + skipping = iteration.clone + remove_capture(skipping, reference, exclude: true) + prefix = skipping.with(quantifier: Node::Quantifier.new(0, -1, reluctant: internal ? quantifier.reluctant? : capture_first)) + update = iteration.clone + require_group_participation(update, reference) + suffix = skipping.with(quantifier: Node::Quantifier.new(0, -1, reluctant: quantifier.reluctant?)) + # Put skips before or after each update according to the original branch + # order. Every successful cycle includes the capture, so JS retains it. + ends_with_capture = !internal && capture_first != quantifier.reluctant? + cycle = ends_with_capture ? Node.new('(?:', prefix, update, ')') : Node.new('(?:', update, suffix, ')') + cycle.quantify(1, -1, reluctant: quantifier.reluctant?) + present = ends_with_capture ? Node.new(cycle, suffix) : Node.new(prefix, cycle) + absent = skipping.update(quantifier: quantifier) + replace_loop(node, [present], absent, quantifier.reluctant?) + node.walk do |child| + make_condition_dynamic(child) if child.conditional? && child.reference == reference + end + end + + # Present [/a/], absent /b/: greedy => /(?:a|b)/; lazy => /(?:b|a)/. + def replace_loop(node, present_branches, absent, reluctant) + branches = reluctant ? [absent, *present_branches] : [*present_branches, absent] + node.update(type: :plain, reference: nil, quantifier: nil, children: [alternate(branches)]) + end + + # \1 => /(?:[\s\S]*)/ for conservative length/character analysis only; + # this approximation is not emitted as the converted regexp. + def analysis_source(node) + return node if node.is_a?(String) + + case node.type + when :backref, :capture_check, :capture_backref, :multiplex_backref + '(?:[\s\S]*)' + when :conditional, :capture_conditional + '(?:' + node.conditional_branches.map { |child| analysis_source(child) }.join('|') + ')' + else + if node.alternation? + branches = node.children.reject { |child| unmatchable?(child) } + '(?:' + branches.map { |child| analysis_source(child).sub(/\A\|/, '') }.join('|') + ')' + else + node.children.map { |child| analysis_source(child) }.join + end + end + node.quantifier.to_s + end + + # /a?b/ => {'a', 'b'}; /^b/ => {'b'} since the anchor consumes nothing. + def first_characters(expression) + return CharacterSet.new if Node::Quantifier.from_expression(expression)&.zero? + + case expression.type + when :literal + CharacterSet.of_string(expression.text[0].to_s) + when :assertion, :anchor + CharacterSet.new + when :group, :expression + result = nil + expression.expressions.each do |child| + characters = first_characters(child) + unless characters.empty? + result = result ? result | characters : characters + end + break unless child.match_length.min.zero? + end + result || CharacterSet.new + when :meta + if expression.token == :dot + CharacterSet.unicode + else + expression.expressions.map { |child| first_characters(child) }.reject(&:empty?).reduce(:|) || CharacterSet.new + end + else + CharacterSet.of_expression(expression) + end + end + + # For capture 1, (a) => (?:a) with exclude: false; with exclude: true, + # (a) => (?!) and (a)? => an empty sequence. + def remove_capture(node, reference, exclude:) + return unless node.is_a?(Node) + + if node.type == :captured_group && node.reference == reference + if exclude + node.update(type: :plain, reference: nil, children: [node.optional? ? '' : '(?!)'], quantifier: nil) + else + node.make_passive + end + else + node.children.each { |child| remove_capture(child, reference, exclude: exclude) } + end + end +end diff --git a/lib/js_regex/second_pass/resolve_capture_references.rb b/lib/js_regex/second_pass/resolve_capture_references.rb new file mode 100644 index 0000000..218806b --- /dev/null +++ b/lib/js_regex/second_pass/resolve_capture_references.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +# Resolve references to a Ruby capture represented by several JavaScript groups +# after loop expansion or subexpression calls. Backreferences use the latest +# participating copy; conditionals check whether any copy participated. +# Duplicate names try participating values newest first and commit to the first +# match, even if the rest of the pattern later fails. +# Optional nonempty captures can also be tested here, keeping their conditions +# in place so that greedy and lazy choices retain their original priority. +# Example: /(?a)(?b)\k/ => /(a)(b)(?:\2|(?!\2)\1)/. +class JsRegex::SecondPass::ResolveCaptureReferences < JsRegex::SecondPass + def call + @group_paths = {} + @groups = {} + @optional_groups = potentially_absent_groups + tree.walk(trace: true) do |node, event, path| + if event == :exit + if node.type == :captured_group + @group_paths[node.reference] = path.dup + @groups[node.reference] = node + end + next + end + case node.type + when :conditional + if optional_consuming_capture?(node.reference, path) + make_condition_dynamic(node) + resolve_condition(node, path) + end + when :multiplex_backref then resolve_multiplexed_backreference(node, path) + when :capture_backref then resolve_backreference(node, path) + when :capture_conditional then resolve_condition(node, path) + end + end + end + + private + + # Three mandatory candidates [3, 2, 1] => /(?:\3|(?!\3)(?:\2|(?!\2)\1))/; + # a failed continuation cannot make a matching candidate give way to an older one. + def resolve_multiplexed_backreference(node, path) + references = node.children.select { |reference| @group_paths.key?(reference.reference) } + alternatives = references.map { |reference| guard_named_reference(reference, path) } + choice = alternatives.reverse_each.reduce do |fallback, preferred| + # Nest fallbacks so each candidate needs only one exclusion check. + alternate([preferred, Node.new('(?!', preferred.clone, ')', fallback)]) + end + node.update(type: :plain, children: [choice || Node.new('(?!)')]) + end + + # For an optional nullable capture, \1 becomes a condition equivalent to + # (?(1)\1|(?!)); a later pass expands that condition. + def guard_named_reference(reference, path) + return reference if required_capture?(reference.reference, path) + + if @groups.fetch(reference.reference).marked_nonempty? + Node.new(nonempty_capture_check(reference.reference, true), reference) + else + Node.new('(?:', Node.new('(?:', reference, ')'), Node.new('(?:(?!))'), ')', + reference: reference.reference, type: :conditional) + end + end + + # Capture 1 in /(a)?(?(1)b|c)/ => true; in /(a*)?(?(1)b|c)/ => false, + # since an empty capture cannot be distinguished from absence by its value. + def optional_consuming_capture?(reference, path) + group = @groups[reference] + return false unless group&.marked_nonempty? && !required_capture?(reference, path) + + # Captures inside lookarounds can belong to atomic-group emulation. Leave + # those under the existing conditional permutation rules. + @group_paths[reference].none? do |ancestor| + ancestor.positive_assertion? || ancestor.atomic? + end + end + + # history[2,1] => \2 when capture 2 is required; an empty history => /(?!)/. + def resolve_backreference(node, path) + references = node.children.select { |reference| @group_paths.key?(reference.reference) } + if references.empty? + node.update(type: :plain, children: ['(?!)']) + elsif required_capture?(references.first.reference, path) + node.update(type: :plain, children: [references.first]) + else + branches = [] + absent_checks = [] + references.each do |reference| + branches << Node.new(*absent_checks, nonempty_capture_check(reference.reference, true), reference) + absent_checks += [nonempty_capture_check(reference.reference, false)] + break unless @optional_groups.include?(reference.reference) + end + node.update(type: :plain, children: [alternate(branches)]) + end + end + + # Capture 1 at \1 in /(?:(a)\1)?/ => true; in /(a)?\1/ => false. + def required_capture?(reference, path) + !@optional_groups.include?(reference) || required_on_path?(@group_paths[reference], path) + end + + # After a required capture 1, /(?(1)b|c)/ => /(?:(?:b))/. + def resolve_condition(node, path) + references = node.condition_references + if references.any? { |reference| required_on_path?(@group_paths[reference], path) } + select_conditional_branch(node, true) + else + true_branch, false_branch = node.conditional_branches + present = alternate(references.map { |reference| nonempty_capture_check(reference, true) }) + absent = references.map { |reference| nonempty_capture_check(reference, false) } + node.update(type: :plain, children: [alternate([ + Node.new(present, true_branch), + Node.new(*absent, false_branch || Node.new), + ])]) + end + end +end diff --git a/lib/js_regex/second_pass/substitute_root_level_keep_mark.rb b/lib/js_regex/second_pass/substitute_root_level_keep_mark.rb new file mode 100644 index 0000000..257aa59 --- /dev/null +++ b/lib/js_regex/second_pass/substitute_root_level_keep_mark.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# Replace supported \K marks with lookbehind for the preceding part of the branch. +# That text must still match, but is excluded from the returned match. +# Example: /a\Kb/ => /(?<=a)b/. +class JsRegex::SecondPass::SubstituteRootLevelKeepMark < JsRegex::SecondPass + def call + substitute_in(tree) + end + + # /a\Kb\Kc/ => /(?<=ab)c/: only the last mark sets the match start. + def substitute_in(node) + node.children.each do |child| + substitute_in(child) if child.is_a?(Node) + end + + keep_mark_index = node.children.rindex { |child| child.is_a?(Node) && child.type == :keep_mark } + return unless keep_mark_index + + pre = node.children[0...keep_mark_index].reject { |child| child.is_a?(Node) && child.type == :keep_mark } + separator = pre.first == '|' ? [pre.shift] : [] + post = node.children[(keep_mark_index + 1)..-1] + lookbehind = Node.new('(?<=', *pre, ')') + node.update(children: [*separator, lookbehind, *post]) + end +end diff --git a/lib/js_regex/utils.rb b/lib/js_regex/utils.rb index 8e69ab8..4bf3cad 100644 --- a/lib/js_regex/utils.rb +++ b/lib/js_regex/utils.rb @@ -1,3 +1,7 @@ +# Utilities for analyzing Regexp::Expression::* instances emitted by regexp_parser + +require_relative 'utils/captures' +require_relative 'utils/char_sets' require_relative 'utils/char_types' require_relative 'utils/escapes' require_relative 'utils/literals' diff --git a/lib/js_regex/utils/captures.rb b/lib/js_regex/utils/captures.rb new file mode 100644 index 0000000..90b8d52 --- /dev/null +++ b/lib/js_regex/utils/captures.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +class JsRegex + module Utils + module Captures + # Unlike assertions and backreferences, these expressions can match an + # empty string at every input position, regardless of surrounding text. + def self.nullable?(expression, quantified = false) + return true if quantified && JsRegex::Node::Quantifier.from_expression(expression)&.optional? + + case expression.type + when :literal + expression.text.empty? + when :group, :expression + !expression.terminal? && expression.token != :absence && + expression.expressions.all? { |child| nullable?(child, true) } + when :meta + expression.token == :alternation && expression.expressions.any? { |child| nullable?(child, true) } + else + false + end + end + + # A participating capture ignores its own quantifier. Treat backreferences + # conservatively rather than following potentially recursive references. + def self.nonempty?(expression, quantified = false) + return false if quantified && JsRegex::Node::Quantifier.from_expression(expression)&.optional? + + case expression.type + when :literal + !expression.text.empty? + when :escape, :property, :nonproperty, :set, :type + true + when :group, :expression + !expression.terminal? && expression.token != :absence && + expression.expressions.any? { |child| nonempty?(child, true) } + when :meta + expression.token == :dot || + (expression.token == :alternation && expression.expressions.all? { |child| nonempty?(child, true) }) + else + false + end + end + end + end +end diff --git a/lib/js_regex/utils/char_sets.rb b/lib/js_regex/utils/char_sets.rb new file mode 100644 index 0000000..573fd9f --- /dev/null +++ b/lib/js_regex/utils/char_sets.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'character_set' + +class JsRegex + module Utils + module CharSets + def self.convert(expression, context) + # Calculate the set without case folding so the original casing is + # preserved when the JavaScript regexp already has the i flag. + neutral_expression = expression.dup + neutral_expression.options[:i] = false + unless neutral_expression.terminal? + neutral_expression.each_expression(true) { |exp| exp.options[:i] = false } + end + content = CharacterSet.of_expression(neutral_expression) + + if expression.case_insensitive? && !context.case_insensitive_root + content = content.case_insensitive + elsif !expression.case_insensitive? && context.case_insensitive_root + context.warn_of("Dropped unsupported nested case-sensitive character set '#{expression}' "\ + "at index #{expression.ts}") + end + + if context.es_2015_or_higher? + context.enable_u_option if content.astral_part? + content.to_s(format: 'es6', in_brackets: true) + else + content.to_s_with_surrogate_ranges + end + end + end + end +end diff --git a/lib/js_regex/utils/literals.rb b/lib/js_regex/utils/literals.rb index 79e19af..04e29ed 100644 --- a/lib/js_regex/utils/literals.rb +++ b/lib/js_regex/utils/literals.rb @@ -10,7 +10,7 @@ module Literals class << self def convert_data(data, context) - if !context.u? && data =~ ASTRAL_PLANE_CODEPOINT_PATTERN + if !context.u? && data.match?(ASTRAL_PLANE_CODEPOINT_PATTERN) if context.enable_u_option escape_incompatible_bmp_literals(data) else @@ -27,11 +27,12 @@ def escape_incompatible_bmp_literals(data) private - LITERAL_REQUIRING_ESCAPE_PATTERN = /[\/\f\n\r\t\v]/.freeze + LITERAL_REQUIRING_ESCAPE_PATTERN = /[\/\f\n\r\t\v\u2028\u2029]/.freeze ESCAPES = Hash.new { |h, k| raise KeyError, "#{h}[#{k.inspect}]" } .merge("\f\n\r\t\v".chars.to_h { |c| [c, Regexp.escape(c)] }) .merge('/' => '\\/') + .merge("\u2028" => '\\u2028', "\u2029" => '\\u2029') def convert_astral_data(data) data.each_char.each_with_object(JsRegex::Node.new) do |char, node| diff --git a/lib/js_regex/utils/property_map.csv b/lib/js_regex/utils/property_map.csv index c13d7b1..17a5ec1 100644 --- a/lib/js_regex/utils/property_map.csv +++ b/lib/js_regex/utils/property_map.csv @@ -2,9 +2,11 @@ ascii,ASCII asciihexdigit,ASCII_Hex_Digit adlam,Script=Adlam +ahom,Script=Ahom anatolianhieroglyphs,Script=Anatolian_Hieroglyphs armenian,Script=Armenian avestan,Script=Avestan +balinese,Script=Balinese bamum,Script=Bamum bassavah,Script=Bassa_Vah batak,Script=Batak @@ -12,21 +14,28 @@ bengali,Script=Bengali bhaiksuki,Script=Bhaiksuki bidicontrol,Bidi_Control bopomofo,Script=Bopomofo +brahmi,Script=Brahmi braille,Script=Braille buginese,Script=Buginese buhid,Script=Buhid +canadianaboriginal,Script=Canadian_Aboriginal carian,Script=Carian caucasianalbanian,Script=Caucasian_Albanian chakma,Script=Chakma cham,Script=Cham cherokee,Script=Cherokee chorasmian,Script=Chorasmian +closepunctuation,Close_Punctuation connectorpunctuation,Connector_Punctuation control,Control coptic,Script=Coptic cuneiform,Script=Cuneiform cypriot,Script=Cypriot +cyprominoan,Script=Cypro_Minoan cyrillic,Script=Cyrillic +dash,Dash +dashpunctuation,Dash_Punctuation +defaultignorablecodepoint,Default_Ignorable_Code_Point deprecated,Deprecated deseret,Script=Deseret devanagari,Script=Devanagari @@ -38,9 +47,13 @@ elbasan,Script=Elbasan elymaic,Script=Elymaic emojicomponent,Emoji_Component emojimodifier,Emoji_Modifier +emojimodifierbase,Emoji_Modifier_Base enclosingmark,Enclosing_Mark +ethiopic,Script=Ethiopic finalpunctuation,Final_Punctuation +format,Format georgian,Script=Georgian +glagolitic,Script=Glagolitic gothic,Script=Gothic grantha,Script=Grantha greek,Script=Greek @@ -53,6 +66,7 @@ hanunoo,Script=Hanunoo hatran,Script=Hatran hebrew,Script=Hebrew hexdigit,Hex_Digit +hiragana,Script=Hiragana idsbinaryoperator,IDS_Binary_Operator idstrinaryoperator,IDS_Trinary_Operator imperialaramaic,Script=Imperial_Aramaic @@ -61,6 +75,9 @@ inscriptionalpahlavi,Script=Inscriptional_Pahlavi inscriptionalparthian,Script=Inscriptional_Parthian javanese,Script=Javanese joincontrol,Join_Control +kaithi,Script=Kaithi +katakana,Script=Katakana +kawi,Script=Kawi kayahli,Script=Kayah_Li kharoshthi,Script=Kharoshthi khitansmallscript,Script=Khitan_Small_Script @@ -69,7 +86,6 @@ khojki,Script=Khojki khudawadi,Script=Khudawadi lao,Script=Lao lepcha,Script=Lepcha -letternumber,Letter_Number limbu,Script=Limbu lineseparator,Line_Separator lineara,Script=Linear_A @@ -85,8 +101,6 @@ mandaic,Script=Mandaic manichaean,Script=Manichaean marchen,Script=Marchen masaramgondi,Script=Masaram_Gondi -math,Math -mathsymbol,Math_Symbol medefaidrin,Script=Medefaidrin meeteimayek,Script=Meetei_Mayek mendekikakui,Script=Mende_Kikakui @@ -94,10 +108,13 @@ meroiticcursive,Script=Meroitic_Cursive meroitichieroglyphs,Script=Meroitic_Hieroglyphs miao,Script=Miao modi,Script=Modi +modifiersymbol,Modifier_Symbol +mongolian,Script=Mongolian mro,Script=Mro multani,Script=Multani myanmar,Script=Myanmar nabataean,Script=Nabataean +nagmundari,Script=Nag_Mundari nandinagari,Script=Nandinagari newtailue,Script=New_Tai_Lue newa,Script=Newa @@ -115,6 +132,8 @@ oldpersian,Script=Old_Persian oldsogdian,Script=Old_Sogdian oldsoutharabian,Script=Old_South_Arabian oldturkic,Script=Old_Turkic +olduyghur,Script=Old_Uyghur +openpunctuation,Open_Punctuation oriya,Script=Oriya osage,Script=Osage osmanya,Script=Osmanya @@ -136,12 +155,13 @@ rejang,Script=Rejang runic,Script=Runic samaritan,Script=Samaritan saurashtra,Script=Saurashtra +sentenceterminal,Sentence_Terminal separator,Separator -sharada,Script=Sharada shavian,Script=Shavian siddham,Script=Siddham signwriting,Script=SignWriting sinhala,Script=Sinhala +softdotted,Soft_Dotted sogdian,Script=Sogdian sorasompeng,Script=Sora_Sompeng soyombo,Script=Soyombo @@ -149,20 +169,26 @@ spaceseparator,Space_Separator sundanese,Script=Sundanese sylotinagri,Script=Syloti_Nagri syriac,Script=Syriac +tagalog,Script=Tagalog tagbanwa,Script=Tagbanwa taile,Script=Tai_Le taitham,Script=Tai_Tham taiviet,Script=Tai_Viet +takri,Script=Takri tamil,Script=Tamil -tangut,Script=Tangut +tangsa,Script=Tangsa +terminalpunctuation,Terminal_Punctuation thaana,Script=Thaana thai,Script=Thai tibetan,Script=Tibetan tifinagh,Script=Tifinagh tirhuta,Script=Tirhuta titlecaseletter,Titlecase_Letter +toto,Script=Toto ugaritic,Script=Ugaritic vai,Script=Vai +variationselector,Variation_Selector +vithkuqi,Script=Vithkuqi wancho,Script=Wancho warangciti,Script=Warang_Citi whitespace,White_Space diff --git a/spec/acceptance_spec.rb b/spec/acceptance_spec.rb index 8d45596..467437f 100644 --- a/spec/acceptance_spec.rb +++ b/spec/acceptance_spec.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -require 'spec_helper' require 'uri' describe JsRegex do diff --git a/spec/lib/js_regex/conversion_spec.rb b/spec/lib/js_regex/conversion_spec.rb index 07b2336..7136f39 100644 --- a/spec/lib/js_regex/conversion_spec.rb +++ b/spec/lib/js_regex/conversion_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'spec_helper' - describe JsRegex::Conversion do describe '::of' do let(:return_value) { described_class.of(/foo/i) } diff --git a/spec/lib/js_regex/converter/anchor_converter_spec.rb b/spec/lib/js_regex/converter/anchor_converter_spec.rb index 53759a7..453ee5e 100644 --- a/spec/lib/js_regex/converter/anchor_converter_spec.rb +++ b/spec/lib/js_regex/converter/anchor_converter_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'spec_helper' - describe JsRegex::Converter::AnchorConverter do it 'translates the beginning-of-string anchor "\A"' do expect(/\A\w/).to\ @@ -45,8 +43,8 @@ it 'uses a unicode-awary replacement for "\b" on ES2018+', targets: [ES2018] do expect(/\w\b/).to\ - become('\w(?:(?<=[\d\p{L}\p{M}\p{Pc}])(?=[^\d\p{L}\p{M}\p{Pc}]|$)'\ - '|(?<=[^\d\p{L}\p{M}\p{Pc}]|^)(?=[\d\p{L}\p{M}\p{Pc}]))') + become('\w(?:(?<=[\p{N}\p{L}\p{M}\p{Pc}])(?=[^\p{N}\p{L}\p{M}\p{Pc}]|$)'\ + '|(?<=[^\p{N}\p{L}\p{M}\p{Pc}]|^)(?=[\p{N}\p{L}\p{M}\p{Pc}]))') .with_options('u') .and keep_matching('abc tür', with_results: %w[c r]) end @@ -60,12 +58,30 @@ it 'uses a unicode-awary replacement for "\B" on ES2018+', targets: [ES2018] do expect(/\w\B/).to\ - become(/\w(?<=[\d\p{L}\p{M}\p{Pc}])(?=[\d\p{L}\p{M}\p{Pc}])/) + become('\\w(?:(?<=[\\p{N}\\p{L}\\p{M}\\p{Pc}])(?=[\\p{N}\\p{L}\\p{M}\\p{Pc}])'\ + '|(?<=[^\\p{N}\\p{L}\\p{M}\\p{Pc}]|^)(?=[^\\p{N}\\p{L}\\p{M}\\p{Pc}]|$))') .with_options('u') .and keep_matching('abc tür', with_results: %w[a b t]) # ^ this is correct; both in Ruby and JS, \w is is ascii-only by default end + it 'matches non-boundaries between nonword characters and string edges', targets: [ES2018] do + expect(/\B/).to keep_matching('', '!', '!!', ' ') + .and keep_not_matching('a') + expect(/\A\B!\B\z/).to keep_matching('!') + .and keep_not_matching('a', '!!') + expect(/!\B!/).to keep_matching('!!') + .and keep_not_matching('!a!') + end + + it 'treats Unicode numbers as word characters at boundaries', targets: [ES2018] do + %w[١ 9 𝟡 Ⅻ ½].each do |char| + expect(/\A\b#{char}\b\z/).to keep_matching(char) + expect(/\A\B#{char}\z/).to keep_not_matching(char) + expect(/#{char}\B#{char}/).to keep_matching(char * 2) + end + end + it 'drops the previous match anchor "\G" with warning' do expect(/(foo)\G/).to\ become(/(foo)/).with_warning diff --git a/spec/lib/js_regex/converter/assertion_converter_spec.rb b/spec/lib/js_regex/converter/assertion_converter_spec.rb index 58fe106..1a76f04 100644 --- a/spec/lib/js_regex/converter/assertion_converter_spec.rb +++ b/spec/lib/js_regex/converter/assertion_converter_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'spec_helper' - describe JsRegex::Converter::AssertionConverter do it 'preserves positive lookaheads' do expect(/a(?=b)/i).to stay_the_same.and keep_matching('aAb', with_results: %w[A]) diff --git a/spec/lib/js_regex/converter/backreference_converter_spec.rb b/spec/lib/js_regex/converter/backreference_converter_spec.rb index 4ed5d55..940ff2f 100644 --- a/spec/lib/js_regex/converter/backreference_converter_spec.rb +++ b/spec/lib/js_regex/converter/backreference_converter_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'spec_helper' - describe JsRegex::Converter::BackreferenceConverter do it 'preserves traditional numeric backreferences' do expect(/(a)(b)(c)\2/) @@ -72,6 +70,20 @@ .and keep_not_matching('abc') end + it 'delimits converted numeric references before literal digits' do + expect(/\A(a)\k<1>0\z/).to become('^(a)\\1(?:)0$') + .and keep_matching('aa0') + .and keep_not_matching("a\b", 'aa') + expect(/\A(a)\k<-1>2\z/).to keep_matching('aa2') + expect(/\A(?a)\k0\z/).to keep_matching('aa0') + end + + it 'delimits references after conditional renumbering' do + expect(/\A(a)(?(1)b)\k<1>0\z/) + .to keep_matching('aba0') + .and keep_not_matching('ab0', "ab\b") + end + it 'marks backrefs for SecondPass conversion' do backref = Regexp::Parser.parse(/(a)\1/).last @@ -236,6 +248,31 @@ become('(foo)(foo)(bar)\3') end + it 'counts every nested capture inserted by a call' do + expect(/\A(a(b))\g<1>(c)\3\z/) + .to become('^(a(b))(a(b))(c)\\5$') + .and keep_matching('ababcc') + .and keep_not_matching('ababcb') + expect(/\A(a(b))\g<1>(?>c)\z/) + .to keep_matching('ababc') + .and keep_not_matching('ababb') + end + + it 'updates references inside copied groups and repeated calls' do + expect(/\A(a(b)\2)\g<1>\g<1>(c)\3\z/) + .to keep_matching('abbabbabbcc') + .and keep_not_matching('abbabbabbcb') + end + + it 'restores counting after nested calls and forward calls' do + expect(/\A\g<1>(a(b))(c)\3\z/) + .to keep_matching('ababcc') + .and keep_not_matching('ababcb') + expect(/\A(a)(b\g<1>)\g<2>(c)\3\z/) + .to keep_matching('ababacc') + .and keep_not_matching('ababaca') + end + it 'keeps 5 levels of recursive calls with warning' do expect(/a|b\g<0>/).to\ become(/a|b(?:a|b(?:a|b(?:a|b(?:a|b(?:a|b)))))/) @@ -273,6 +310,17 @@ .and keep_not_matching('ababc', 'acabc') end + it 'retains a forward call capture when the original optional group is skipped' do + expect(/\A\g<1>(a)?\1\z/).to keep_matching('aa', 'aaa') + .and keep_not_matching('a', '') + expect(/\A\g<1>([ab])?\1\z/).to keep_matching('aa', 'bb', 'abb', 'baa') + .and keep_not_matching('a', 'b', 'aba', 'bab') + end + + it 'preserves greedy optional captures after forward calls' do + expect(/\A\g<1>(a)?\1/).to keep_capturing('aaa' => 'aaa') + end + it 'uses the correct local flags / options (those that apply to the target)' do expect(/(a)(?i)\g<1>/).to\ become(/(a)(a)/) @@ -284,7 +332,7 @@ context 'when handling multiplexed named groups' do it 'converts named backreferences to multiplexed groups as alternations', targets: [ES2009, ES2015] do expect(/(?a)(?b)\k/).to\ - become(/(a)(b)(?:\1|\2)/) + become(/(a)(b)(?:\2|(?!\2)\1)/) .and keep_matching('aba', 'abb') .and keep_not_matching('abc', 'ab') end @@ -292,18 +340,22 @@ it 'converts multiplexed named groups to numbered groups on ES2018+', targets: [ES2018] do # All named groups are converted to numbered groups, even on ES2018+ expect(/(?a)(?b)\k/).to\ - become(/(a)(b)(?:\1|\2)/) + become(/(a)(b)(?:\2|(?!\2)\1)/) .and keep_matching('aba', 'abb') .and keep_not_matching('abc', 'ab') end it 'handles multiple multiplexed groups', targets: [ES2009, ES2015] do expect(/(?a)(?b)(?c)\k/).to\ - become(/(a)(b)(c)(?:\1|\3)/) + become(/(a)(b)(c)(?:\3|(?!\3)\1)/) .and keep_matching('abca', 'abcc') .and keep_not_matching('abcb', 'abcd') end + it 'tries later duplicate names before earlier names' do + expect(/(?a)(?aa)\k/).to keep_capturing('aaaaa' => 'aaaaa') + end + it 'handles multiplexed groups with alternation', targets: [ES2009, ES2015] do expect(/(?foo)|(?bar)/).to\ become(/(foo)|(bar)/) @@ -318,13 +370,53 @@ .and keep_not_matching('ab') end + it 'rejects references to captures that did not participate' do + expect(/\A(?:(?a)|(?b))\k\z/) + .to keep_matching('aa', 'bb') + .and keep_not_matching('a', 'b', 'ab', 'ba', '') + end + + it 'distinguishes absent captures from participating empty captures' do + expect(/\A(?a)?(?b)?\k\z/) + .to keep_matching('aa', 'bb', 'aba', 'abb') + .and keep_not_matching('', 'a', 'b', 'ab') + expect(/\A(?:(?)|(?a))\k\z/) + .to keep_matching('', 'aa') + .and keep_not_matching('a') + end + + it 'preserves greedy and lazy matching with optional duplicate names' do + [/\A(?a)?(?a)?\k/, /\A(?a)??(?a)?\k/, + /\A(?a)*(?aa)?\k/].each do |regexp| + examples = %w[aa aaa aaaaaaa].to_h { |str| [str, regexp.match(str).to_a] } + expect(regexp).to keep_capturing(examples) + end + end + + it 'rejects absent duplicate captures from successful negative assertions' do + expect(/\A(?!(?a))(?b)\k\z/).to keep_matching('bb') + .and keep_not_matching('b', 'a', 'ab') + end + it 'handles multiplexed groups in x-mode', targets: [ES2009, ES2015] do expect(/(? a ) (? b ) \k /x).to\ - become(/(a)(b)(?:\1|\2)/) + become(/(a)(b)(?:\2|(?!\2)\1)/) .and keep_matching('aba', 'abb') .and keep_not_matching('abc') end + + it 'renumbers multiplexed references when expanding conditionals' do + expect(/\A(?a)(?b)(?c)?(?()d|e)\k\z/) + .to keep_matching('abcda', 'abcdb', 'abea', 'abeb') + .and keep_not_matching('abcd', 'abe', 'abec') + end + + it 'guards multiplexed captures inside conditional branches' do + expect(/\A(?a)?(?()(?b)|(?c))\k\z/) + .to keep_matching('abb', 'cc') + .and keep_not_matching('ab', 'c', 'abc', 'ac') + end end context 'when backreferences point to non-participating groups' do @@ -349,26 +441,33 @@ it 'handles named backreference to non-participating branch (?a)|(?)\k' do expect(/(?a)|(?)\k/).to\ - become(/(a)|()(?:\1|\2)/) - .and keep_matching('a', '') + keep_matching('a', '') end it 'handles backreferences to optional groups from branches' do # this has zero-length matches on 'b' and '' - expect(/(a)?|b\1/).to stay_the_same.and keep_matching('a', 'b', '') + expect(/(a)?|b\1/).to keep_matching('a', 'b', '') end - # TODO: https://github.com/jaynetics/js_regex/issues/26 - # Handling this would require generating a lot of permutations. - # it 'handles backreferences to nested groups on branches' do - # expect(/((a)|b)+\2/).to\ - # become(/((a)|b)+\2/) # TODO - # .and keep_matching('aa', 'aba', 'baa') - # .and keep_not_matching('a', 'b', 'ab', 'ba') - # end - it 'handles backreference to group that may capture empty string (a?)b\1' do expect(/(a?)b\1/).to stay_the_same.and keep_matching('aba', 'b') end + + it 'retains nested captures skipped by later iterations' do + expect(/((a)|b)+\2/).to keep_matching('aa', 'aba', 'baa') + .and keep_not_matching('a', 'b', 'bb', 'ab', 'ba') + .and keep_capturing('aba' => 'aba') + end + + it 'requires a referenced capture among several repeated alternatives' do + expect(/((a)|(b)|c|d)+\3/).to keep_matching('bb', 'bcb', 'bdb') + .and keep_not_matching('c', 'd', 'bc') + end + + it 'allows branches or optional backreferences to skip a retained capture' do + expect(/((a)|(b)|c|d)+(?:e|\3)/).to keep_matching('bb', 'be', 'ce', 'bcb') + .and keep_not_matching('c', 'd', 'bc') + expect(/\A((a)|b)+\2?\z/).to keep_matching('a', 'b', 'ab', 'aba') + end end end diff --git a/spec/lib/js_regex/converter/base_spec.rb b/spec/lib/js_regex/converter/base_spec.rb index 4fe1f04..327f847 100644 --- a/spec/lib/js_regex/converter/base_spec.rb +++ b/spec/lib/js_regex/converter/base_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'spec_helper' - describe JsRegex::Converter::Base do describe '#warn_of_unsupported_feature' do it 'adds a warning with token class, subtype, data and index' do @@ -90,6 +88,21 @@ expect(/a+?/).to stay_the_same end + it 'preserves lazy intervals that contain a comma', + if: Gem::Version.new(Regexp::Parser::VERSION) > Gem::Version.new('2.13.1') do + expect(/a{1,3}?/).to stay_the_same + .and keep_matching('aaa', with_results: %w[a a a]) + expect(/a{2,}?/).to stay_the_same + .and keep_matching('aaaa', with_results: %w[aa aa]) + expect(/a{,3}?/).to become(/a{0,3}?/) + end + + it 'keeps nonempty lazy intervals from introducing empty captures', + if: Gem::Version.new(Regexp::Parser::VERSION) > Gem::Version.new('2.13.1') do + expect(/\A(a{1,3}?)*\1\z/).to keep_matching('aa', 'aaa', 'aaaa') + .and keep_not_matching('', 'a') + end + it 'preserves set quantifiers' do expect(/[a-z]+?/).to stay_the_same end @@ -125,6 +138,12 @@ become(/(?:(?=((?:a|b)+))\1)/) end + it 'numbers the wrapper before nested capturing groups' do + expect(/\A(a(b))++\2\z/) + .to keep_matching('abb', 'ababb') + .and keep_not_matching('ab', 'abab') + end + it 'takes into account preceding active groups for the backreference' do expect(/(a)(b)(c)_d++/).to\ become(/(a)(b)(c)_(?:(?=(d+))\4)/) @@ -152,6 +171,11 @@ become(/(?:a{4}){6}/) end + it 'keeps a question mark after an exact count optional' do + expect(/a{3}?/).to become(/(?:a{3})?/).and keep_matching('') + expect(/(?:a{1,3})?/).to stay_the_same + end + it 'wraps adjacent/multiplicative ranges ({x,y}) in passive groups' do expect(/a{2,4}{3,6}/).to\ become(/(?:a{2,4}){3,6}/) @@ -179,6 +203,10 @@ expect(/a\G++b/).to\ become(/ab/).with_warning end + + it 'does not reserve captures for dropped possessive tokens' do + expect(/a\G++(b)\1/).to become(/a(b)\1/).with_warning + end end end @@ -218,29 +246,29 @@ before { allow(converter).to receive(:context).and_return(context) } it 'returns the contents wrapped in a backreferenced lookahead' do - result = converter.send(:wrap_in_backrefed_lookahead, %w[foo bar]) + result = converter.send(:wrap_in_backrefed_lookahead) { %w[foo bar] } expect(result.to_s).to eq('(?:(?=(foobar))\\1)') expect(result.children[3].type).to eq :backref expect(result.children[3].children).to eq %w[\\1] end it 'increases the count of captured groups' do - expect { converter.send(:wrap_in_backrefed_lookahead, 'foo') } + expect { converter.send(:wrap_in_backrefed_lookahead) { 'foo' } } .to change { context.send(:capturing_group_count) }.from(0).to(1) end it 'increases the new_capturing_group_position for any following group' do context.capture_group expect(context.new_capturing_group_position(1)).to eq(1) - expect(converter.send(:wrap_in_backrefed_lookahead, 'foo').to_s).to include '\2' + expect(converter.send(:wrap_in_backrefed_lookahead) { 'foo' }.to_s).to include '\2' expect(context.new_capturing_group_position(2)).to eq(3) - expect(converter.send(:wrap_in_backrefed_lookahead, 'foo').to_s).to include '\3' + expect(converter.send(:wrap_in_backrefed_lookahead) { 'foo' }.to_s).to include '\3' end it 'doesnt increase the new_capturing_group_position of preceding groups' do context.capture_group expect(context.new_capturing_group_position(1)).to eq(1) - expect(converter.send(:wrap_in_backrefed_lookahead, 'foo').to_s).to include '\2' + expect(converter.send(:wrap_in_backrefed_lookahead) { 'foo' }.to_s).to include '\2' expect(context.new_capturing_group_position(1)).to eq(1) end end diff --git a/spec/lib/js_regex/converter/conditional_converter_spec.rb b/spec/lib/js_regex/converter/conditional_converter_spec.rb index 3da8efd..1b48b90 100644 --- a/spec/lib/js_regex/converter/conditional_converter_spec.rb +++ b/spec/lib/js_regex/converter/conditional_converter_spec.rb @@ -1,9 +1,7 @@ # frozen_string_literal: true -require 'spec_helper' - describe JsRegex::Converter::ConditionalConverter do - # see second_pass_spec.rb for tests of the final results + # See second_pass/alternate_conditional_permutations_spec.rb for expansion details. it 'marks conditionals for SecondPass conversion' do conditional = Regexp::Parser.parse(/(a)(?(1)b|c)/)[1] @@ -20,7 +18,7 @@ it 'drops the condition part without warning' do expect(/(a)(?(1)b|c)/).to\ - become(/(?:(a){0}(?:(?:b){0}(?:c)))|(?:(a)(?:(?:b)(?:c){0}))/) + become(/(?:((?!)a)(?:(?:c)))|(?:(a)(?:(?:b)))/) end it 'drops unknown conditional expressions with warning' do diff --git a/spec/lib/js_regex/converter/context_spec.rb b/spec/lib/js_regex/converter/context_spec.rb index f7362a1..c06c937 100644 --- a/spec/lib/js_regex/converter/context_spec.rb +++ b/spec/lib/js_regex/converter/context_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'spec_helper' - describe JsRegex::Converter::Context do let(:context) { described_class.new } @@ -61,13 +59,13 @@ end end - describe '#increment_local_capturing_group_count' do + describe '#capture_group(added: true)' do it 'adds to added_capturing_groups_after_group based on current position' do context.capture_group - context.increment_local_capturing_group_count + context.capture_group(added: true) context.capture_group - context.increment_local_capturing_group_count - context.increment_local_capturing_group_count + context.capture_group(added: true) + context.capture_group(added: true) expect(context.send(:added_capturing_groups_after_group)) .to eq(1 => 1, 2 => 2) end diff --git a/spec/lib/js_regex/converter/escape_converter_spec.rb b/spec/lib/js_regex/converter/escape_converter_spec.rb index 23bd940..400821c 100644 --- a/spec/lib/js_regex/converter/escape_converter_spec.rb +++ b/spec/lib/js_regex/converter/escape_converter_spec.rb @@ -9,8 +9,6 @@ # # -require 'spec_helper' - describe JsRegex::Converter::EscapeConverter do it 'lets backslashes pass through' do expect(/\\/).to stay_the_same.and keep_matching('\\', with_results: %w[\\]) diff --git a/spec/lib/js_regex/converter/freespace_converter_spec.rb b/spec/lib/js_regex/converter/freespace_converter_spec.rb index fdabaa3..e7dd4c2 100644 --- a/spec/lib/js_regex/converter/freespace_converter_spec.rb +++ b/spec/lib/js_regex/converter/freespace_converter_spec.rb @@ -9,8 +9,6 @@ # # -require 'spec_helper' - describe JsRegex::Converter::FreespaceConverter do context 'when extended mode is set' do it 'drops comments and whitespace' do diff --git a/spec/lib/js_regex/converter/group_converter_spec.rb b/spec/lib/js_regex/converter/group_converter_spec.rb index 624460f..7217879 100644 --- a/spec/lib/js_regex/converter/group_converter_spec.rb +++ b/spec/lib/js_regex/converter/group_converter_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'spec_helper' - describe JsRegex::Converter::GroupConverter do it 'preserves capturing groups' do expect(/(abc)/).to stay_the_same.and keep_matching('abc', with_results: %w[abc]) @@ -116,6 +114,15 @@ .and keep_not_matching('ab_1337') end + it 'numbers the wrapper before its nested captures and backreferences' do + expect(/\A(?>a(b))\1\z/) + .to keep_matching('abb') + .and keep_not_matching('ab', 'abab') + expect(/\A(?>a(b)\1)\z/) + .to keep_matching('abb') + .and keep_not_matching('ab') + end + it 'isnt confused by preceding passive groups' do expect(/(?:c)_1(?>33|3)37/).to\ become(/(?:c)_1(?:(?=(33|3))\1)37/) @@ -140,20 +147,44 @@ context 'when dealing with absence groups', if: (Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new('2.4.1')) do - it 'converts simple cases to complimentary alternations' do + it 'converts simple cases to a guarded repetition followed by a short tail' do expect(Regexp.new('1(?~23)4')).to\ - become(/1(?:(?:.|\n){,1}|(?:(?!23)(?:.|\n))*)4/) + become(/1(?:(?:(?!23)(?:.|\n))*(?:.|\n){0,1})4/) .and keep_matching('14', '124', '134', '12224', '13334') .and keep_not_matching('1234', '12234') end it 'can handle fixed quantifications' do expect(Regexp.new('A(?~\d{4})Z')).to\ - become(/A(?:(?:.|\n){,3}|(?:(?!\d{4})(?:.|\n))*)Z/) + become(/A(?:(?:(?!\d{4})(?:.|\n))*(?:.|\n){0,3})Z/) .and keep_matching('AZ', 'A123Z', 'A12X34Z') .and keep_not_matching('A1234Z') end + it 'matches the longest prefix without a complete forbidden pattern' do + expect(/\A(?~\d{4})/).to keep_matching('abcd', with_results: ['abcd']) + expect(/\A(?~\d{4})/).to keep_matching('X1234', with_results: ['X123']) + end + + it 'allows the forbidden pattern to overlap what follows the absence group' do + expect(/\A(?~\d{4})\d\z/).to keep_matching('1234', 'X1234') + .and keep_not_matching('12345') + end + + it 'handles zero-width forbidden expressions without negative quantifier bounds' do + expect(/\A(?~(?=a))\z/).to become('^(?:(?:(?!(?=a))(?:.|\n))*)$') + .and keep_matching('', 'b', 'bbb') + .and keep_not_matching('a', 'ab', 'ba') + expect(/\A(?~(?=a))😋\z/).to keep_matching('😋', 'b😋') + .and keep_not_matching('a😋', 'ba😋') + end + + it 'supports absence groups when another token enables the u flag', targets: [ES2015, ES2018] do + expect(/\A(?~ab)😋\z/) + .to keep_matching('😋', 'x😋', 'xyz😋') + .and keep_not_matching('ab😋', 'xab😋') + end + it 'drops variably quantified cases with warning' do expect(Regexp.new('1(?~2+)3')).to\ become(/13/).with_warning('variable-length absence group content') diff --git a/spec/lib/js_regex/converter/keep_converter_spec.rb b/spec/lib/js_regex/converter/keep_converter_spec.rb index 681ade8..588ec93 100644 --- a/spec/lib/js_regex/converter/keep_converter_spec.rb +++ b/spec/lib/js_regex/converter/keep_converter_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'spec_helper' - describe JsRegex::Converter::KeepConverter do it 'drops the keep mark "\K" with warning', targets: [ES2009, ES2015] do expect(/a\Kb/).to\ @@ -13,6 +11,21 @@ become(/(?<=a)b/) end + it 'uses the last keep mark in a sequence', targets: [ES2018] do + expect(/a\Kb\Kc/).to become(/(?<=ab)c/) + .and keep_matching('abc', with_results: ['c']) + .and keep_not_matching('bc', 'ac') + end + + it 'converts keep marks on every root alternative', targets: [ES2018] do + expect(/a\Kb|c/).to become(/(?<=a)b|c/) + .and keep_matching('abc', with_results: %w[b c]) + expect(/c|a\Kb\Kc/).to become(/c|(?<=ab)c/) + .and keep_matching('abc c', with_results: %w[c c]) + expect(/a\Kb|c\Kd/).to become(/(?<=a)b|(?<=c)d/) + .and keep_matching('ab cd', with_results: %w[b d]) + end + it 'drops nested keep marks on ES2018+', targets: [ES2018] do # This special case that can't be reproduced with a normal lookbehind # because the keep mark works across group boundaries. diff --git a/spec/lib/js_regex/converter/literal_converter_spec.rb b/spec/lib/js_regex/converter/literal_converter_spec.rb index 0909c23..eee4cfc 100644 --- a/spec/lib/js_regex/converter/literal_converter_spec.rb +++ b/spec/lib/js_regex/converter/literal_converter_spec.rb @@ -9,8 +9,6 @@ # # -require 'spec_helper' - describe JsRegex::Converter::LiteralConverter do it 'converts literal newlines into newline escapes' do expect(/ @@ -38,6 +36,11 @@ become(/\t/).and keep_matching(' ', with_results: [' ']) end + it 'escapes Unicode line and paragraph separators in JavaScript literals' do + expect(Regexp.new("\u2028\u2029")).to become('\\u2028\\u2029') + .and keep_matching("a\u2028\u2029b", with_results: ["\u2028\u2029"]) + end + it 'converts literal forward slashes to forward slash escapes' do expect(%r{//}).to\ become('\\/\\/').and keep_matching('a//b', with_results: %w[//]) @@ -95,6 +98,13 @@ become(/1(?:2)34/).and keep_matching('1234', with_results: %w[1234]) end + it 'keeps generated Unicode escapes intact in locally case-insensitive runs' do + expect(Regexp.new("(?i:a\u2028\u2029)")).to become('(?:[aA]\\u2028\\u2029)') + .and keep_matching("A\u2028\u2029", "a\u2028\u2029") + expect(Regexp.new("(?i:\u2028)")).to become('(?:\\u2028)') + .and keep_matching("\u2028") + end + it 'warns for case-sensitive literals in case-insensitive regexes' do expect(/a(?-i)b/i).to\ become(/ab/i).with_warning("nested case-sensitive literal 'b'") diff --git a/spec/lib/js_regex/converter/meta_converter_spec.rb b/spec/lib/js_regex/converter/meta_converter_spec.rb index f67bf42..1c3452b 100644 --- a/spec/lib/js_regex/converter/meta_converter_spec.rb +++ b/spec/lib/js_regex/converter/meta_converter_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'spec_helper' - describe JsRegex::Converter::MetaConverter do it 'replaces the dot meta char so that it keeps matching astral stuff, too' do expect(/a.a/).to\ diff --git a/spec/lib/js_regex/converter/property_converter_spec.rb b/spec/lib/js_regex/converter/property_converter_spec.rb index 31c24a7..4d92923 100644 --- a/spec/lib/js_regex/converter/property_converter_spec.rb +++ b/spec/lib/js_regex/converter/property_converter_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'spec_helper' - describe JsRegex::Converter::PropertyConverter do it 'substitutes the \p{...} property style', targets: [ES2009, ES2015] do expect(/\p{ascii}/).to\ @@ -18,12 +16,18 @@ become('[0-9A-Fa-f]').and keep_matching('añB', with_results: %w[a B]) end - it 'substitutes the negated \p{^...} property style', targets: [ES2009, ES2015] do + it 'substitutes the negated \p{^...} property style', targets: [ES2009] do expect(/\p{^ascii}/).to\ become('(?:[\x80-\uD7FF\uE000-\uFFFF]|[\uD800-\uDBFF][\uDC00-\uDFFF])') .and keep_matching('añB😁', with_results: %w[ñ 😁]) end + it 'uses unicode ranges for expanded astral properties', targets: [ES2015] do + expect(/\p{^ascii}/).to become('[\x80-\uD7FF\uE000-\u{10FFFF}]').with_options('u') + .and keep_matching('ñ😁', with_results: %w[ñ 😁]) + .and keep_not_matching('aB') + end + it 'keeps supported negated properties on ES2018+', targets: [ES2018] do expect(/\p{^ascii}/).to\ become(/\P{ASCII}/).and keep_matching('añB', with_results: %w[ñ]) @@ -51,6 +55,13 @@ .and keep_matching('A B', with_results: %w[ ]) end + it 'keeps astral expansions compatible with other unicode tokens', targets: [ES2015, ES2018] do + expect(/\A\p{Lu}😋\z/).to keep_matching('𐐀😋', 'A😋') + .and keep_not_matching('a😋') + expect(/\A😋\p{Lu}\z/).to keep_matching('😋𐐀', '😋A') + .and keep_not_matching('😋a') + end + it 'uses case-insensitive substitutions if needed' do result = JsRegex.new(/1(?i:\p{lower})2/) expect(result.source).to include 'A-Z' @@ -64,6 +75,6 @@ it 'warns for nested case-sensitive properties' do expect(/(?-i:\p{upper})/i) - .to generate_warning('nested case-sensitive property') + .to generate_warning('nested case-sensitive character set') end end diff --git a/spec/lib/js_regex/converter/set_converter_spec.rb b/spec/lib/js_regex/converter/set_converter_spec.rb index 1c02102..da473e4 100644 --- a/spec/lib/js_regex/converter/set_converter_spec.rb +++ b/spec/lib/js_regex/converter/set_converter_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'spec_helper' - describe JsRegex::Converter::SetConverter do it 'preserves hex escape members' do expect(/[\x41]/).to stay_the_same.and keep_matching('ABC', with_results: %w[A]) @@ -177,6 +175,13 @@ expect(/[a\nb]/).to stay_the_same.and keep_matching("x\ny", with_results: %W[\n]) end + it 'escapes literal Unicode line separators in sets and ranges' do + expect(Regexp.new("[\u2028\u2029]")).to become('[\\u2028\\u2029]') + .and keep_matching("a\u2028\u2029b", with_results: ["\u2028", "\u2029"]) + expect(Regexp.new("[\u2028-\u2029]")).to become('[\\u2028-\\u2029]') + .and keep_matching("\u2028\u2029", with_results: ["\u2028", "\u2029"]) + end + it 'adds case-swapped literal member dupes if subject to a local i-option' do expect(/[a](?i)[a[b]](?-i:[a](?i:[^a-fG-Y]))/) .to keep_matching('aAaZ', with_results: %w[aAaZ]) @@ -190,7 +195,7 @@ it 'warns for locally case-sensitive sets' do expect(/(?-i:[a])/i).to\ - become(/(?:[a])/i).with_warning('nested case-sensitive set') + become(/(?:[a])/i).with_warning('nested case-sensitive character set') end it 'does not add duplicates for literal members that cant be swapped' do diff --git a/spec/lib/js_regex/converter/type_converter_spec.rb b/spec/lib/js_regex/converter/type_converter_spec.rb index ff46ffc..cf7b5c4 100644 --- a/spec/lib/js_regex/converter/type_converter_spec.rb +++ b/spec/lib/js_regex/converter/type_converter_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'spec_helper' - describe JsRegex::Converter::TypeConverter do it 'preserves all types supported by JS in regular mode' do expect(/\d\D\s\S\w\W/).to stay_the_same @@ -30,6 +28,22 @@ .to keep_matching(" ").and keep_not_matching(' ') end + it 'preserves astral letters and digits in unicode types' do + expect(/\A(?u:\w)\z/).to keep_matching('𐐀') + .and keep_not_matching('😋') + expect(/\A(?u:\d)\z/).to keep_matching('𝟡') + .and keep_not_matching('𐐀') + expect(/\A(?u:\w{2})\z/).to keep_matching('𐐀𝟡') + .and keep_not_matching('𐐀', '𐐀𝟡a') + end + + it 'matches whole astral characters in negated unicode types' do + expect(/\A(?u:\W)\z/).to keep_matching('😋') + .and keep_not_matching('𐐀', '𝟡') + expect(/\A(?u:\D)\z/).to keep_matching('😋', '𐐀') + .and keep_not_matching('𝟡') + end + it 'substitutes the hex type "\h" with an equivalent set', targets: [ES2009, ES2015] do expect(/\h+/).to\ become(/[0-9A-Fa-f]+/).and keep_matching('f').and keep_not_matching('x') diff --git a/spec/lib/js_regex/converter/unsupported_token_converter_spec.rb b/spec/lib/js_regex/converter/unsupported_token_converter_spec.rb index abf3839..02d3f9c 100644 --- a/spec/lib/js_regex/converter/unsupported_token_converter_spec.rb +++ b/spec/lib/js_regex/converter/unsupported_token_converter_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'spec_helper' - describe JsRegex::Converter::UnsupportedTokenConverter do it 'drops tokens of unknown classes with warning' do expect([:unknown_class, :some_subtype]).to be_dropped_with_warning diff --git a/spec/lib/js_regex/node_spec.rb b/spec/lib/js_regex/node_spec.rb new file mode 100644 index 0000000..22412ca --- /dev/null +++ b/spec/lib/js_regex/node_spec.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +describe JsRegex::Node do + it 'allows enumerable traversal to stop before visiting later descendants' do + first = described_class.new('a') + later = described_class.new('b') + root = described_class.new(first, later) + expect(later).not_to receive(:walk) + expect(root.walk.any? { |node| node.equal?(first) }).to be(true) + end + + it 'excludes the root from enumerable traversal when requested' do + leaf = described_class.new('a') + child = described_class.new(leaf) + root = described_class.new(child) + expect(root.walk(skip_self: true).to_a).to eq([child, leaf]) + end + + it 'preserves optionality when replacing a reference with failure' do + quantifier = JsRegex::Node::Quantifier.new(0, 1) + node = described_class.new('\\1', type: :backref, reference: 1).update(quantifier: quantifier) + expect(node.make_unmatchable).to equal(node) + expect(node.to_s).to eq('(?!)?') + expect(node.reference).to be_nil + expect(node.quantifier).to equal(quantifier) + end + + it 'distinguishes an empty sequence from an alternation' do + expect(described_class.new.alternation?).to be(false) + expect(described_class.new(described_class.new('a', type: :branch)).alternation?).to be(true) + end +end diff --git a/spec/lib/js_regex/second_pass/alternate_conditional_permutations_spec.rb b/spec/lib/js_regex/second_pass/alternate_conditional_permutations_spec.rb new file mode 100644 index 0000000..aba1f52 --- /dev/null +++ b/spec/lib/js_regex/second_pass/alternate_conditional_permutations_spec.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +describe JsRegex::SecondPass::AlternateConditionalPermutations do + before { stub_const('JsRegex::SecondPass::STEPS', [described_class]) } + + it 'leaves trees without conditionals unchanged' do + expect(/foo/).to stay_the_same + end + + it 'replaces one-branch conditionals with equivalent alternations' do + expect(/-(<)?a(?(1)>)-/).to become('(?:-(){0}a(?:)-)|(?:-(<)a(?:(?:>))-)') + .and keep_matching('-a-', '--') + .and keep_not_matching('--') + end + + it 'replaces two-branch conditionals with equivalent alternations' do + expect(/-(<)?a(?(1)>|b)-/).to become('(?:-(){0}a(?:(?:b))-)|(?:-(<)a(?:(?:>))-)') + .and keep_matching('-ab-', '--') + .and keep_not_matching('-a>-', '--') + end + + it 'replaces named conditionals with equivalent alternations' do + expect(/-(?<)?a(?()>)-/).to become('(?:-(){0}a(?:)-)|(?:-(<)a(?:(?:>))-)') + .and keep_matching('-a-', '--') + .and keep_not_matching('--') + end + + it 'replaces quantified conditionals with equivalent alternations' do + expect(/-(<)?a(?(1)>){3}-/).to become('(?:-(){0}a(?:){3}-)|(?:-(<)a(?:(?:>)){3}-)') + .and keep_matching('-a-', '->>-') + .and keep_not_matching('--') + end + + it 'keeps 0..(n>1) quantifiers of the target group' do + expect(/-(<){,2}a(?(1)>)-/).to become('(?:-(){0}a(?:)-)|(?:-(<){1,2}a(?:(?:>))-)') + .and keep_matching('--', '-<-') + .and keep_not_matching('--') + end + + it 'keeps unbounded and reluctant quantifiers of the target group' do + [/\A(a)*(?(1)b|c)\z/, /\A(a)*?(?(1)b|c)\z/, + /\A(a){0,}(?(1)b|c)\z/].each do |regexp| + expect(regexp).to keep_matching('c', 'ab', 'aab') + .and keep_not_matching('b', 'ac', 'aac') + end + end + + it 'keeps zero-only conditional captures unmatchable in true permutations' do + expect(/\A(a){0}(?(1)b|c)\z/).to keep_matching('c') + .and keep_not_matching('ab', 'b', 'ac') + end + + it 'does not remove mandatory captures in false permutations' do + expect(/\Aa(b)(?(1)c|d)\z/).to keep_matching('abc') + .and keep_not_matching('ad', 'abd', 'ac') + expect(/\A(a)+(?(1)b|c)\z/).to keep_matching('ab', 'aab') + .and keep_not_matching('c', 'ac') + end + + it 'requires enclosing optional groups and alternatives for true conditions' do + expect(/\A(?:(a)|b)?(?(1)c|d)\z/).to keep_matching('ac', 'bd', 'd') + .and keep_not_matching('c', 'bc', 'ad') + end + + it 'replaces successive conditionals with equivalent alternations' do + expect(/(<)?a(?(1)>)(<)?b(?(2)>)/).to keep_matching('ab', 'a', 'b', '') + .and keep_not_matching('a>b', '|>>))-/).to keep_matching('--', '->-', '--b-') + .and keep_not_matching('->-', '--b>>-') + end + + it 'keeps nested capture slots unmatched when discarding their parent' do + expect(/\A(a(b))?(?(1)c|d)(?(2)e|f)(g)\3\z/) + .to keep_matching('abcegg', 'dfgg') + .and keep_not_matching('degg', 'abcfgg', 'dfg', 'abceg') + end + + it 'keeps later reference numbers when discarding nested conditional branches' do + expect(/\A(a)?(?(1)(?(1)b|(c))|(d))(e)\4\z/) + .to keep_matching('abee', 'dee') + .and keep_not_matching('acee', 'abed', 'de', 'ee') + end + + it 'adapts backref numbers in the created alternation branches' do + expect(/()(?(1))\1/).to become('(?:((?!))(?:)\\1)|(?:()(?:(?:))\\2)') + expect(/(a)(b)(?(1)c)(d)\2/) + .to become('(?:((?!)a)(b)(?:)(d)\\2)|(?:(a)(b)(?:(?:c))(d)\\5)') + end +end diff --git a/spec/lib/js_regex/second_pass/analysis_spec.rb b/spec/lib/js_regex/second_pass/analysis_spec.rb new file mode 100644 index 0000000..dad270c --- /dev/null +++ b/spec/lib/js_regex/second_pass/analysis_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +describe JsRegex::SecondPass do + it 'can exclude impossible alternatives when finding absent captures' do + capture = JsRegex::Node.new('(', 'a', ')', type: :captured_group, reference: 1) + root = JsRegex::Node.new(JsRegex::Node.new(capture, type: :branch), + JsRegex::Node.new('|', '(?!)', type: :branch)) + analysis = described_class.new(root) + expect(analysis.send(:potentially_absent_groups)).to eq([1]) + expect(analysis.send(:potentially_absent_groups, reachable_only: true)).to eq([]) + end + + it 'treats captures in dynamic conditional branches as optional in both modes' do + capture = JsRegex::Node.new('(', 'a', ')', type: :captured_group, reference: 1) + root = JsRegex::Node.new(JsRegex::Node.new, capture, JsRegex::Node.new('b'), + type: :capture_conditional) + analysis = described_class.new(root) + expect(analysis.send(:potentially_absent_groups)).to eq([1]) + expect(analysis.send(:potentially_absent_groups, reachable_only: true)).to eq([1]) + end +end diff --git a/spec/lib/js_regex/second_pass/delimit_backrefs_spec.rb b/spec/lib/js_regex/second_pass/delimit_backrefs_spec.rb new file mode 100644 index 0000000..a29a738 --- /dev/null +++ b/spec/lib/js_regex/second_pass/delimit_backrefs_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +describe JsRegex::SecondPass::DelimitBackrefs do + before { stub_const('JsRegex::SecondPass::STEPS', [described_class]) } + + it 'leaves references without a following digit unchanged' do + expect(/(a)\1b/).to stay_the_same + end + + it 'separates a numeric reference from a following literal digit' do + expect(/(a)\1 0/x).to become('(a)\\1(?:)0') + .and keep_matching('aa0') + .and keep_not_matching("a\b", 'aa') + end + + it 'finds following digits across adjacent nodes' do + reference = JsRegex::Node.new('\\12', reference: 12, type: :backref) + tree = JsRegex::Node.new(reference, JsRegex::Node.new, JsRegex::Node.new('3')) + described_class.call(tree) + expect(tree.to_s).to eq('\\12(?:)3') + end + + it 'finds digits through nested containers and dropped nodes' do + reference = JsRegex::Node.new('\\12', reference: 12, type: :backref) + dropped = JsRegex::Node.new('x', type: :dropped) + following = JsRegex::Node.new(JsRegex::Node.new(dropped, JsRegex::Node.new('3'))) + tree = JsRegex::Node.new(reference, following) + described_class.call(tree) + expect(tree.to_s).to eq('\\12(?:)3') + end + + it 'does not separate a reference from its quantifier' do + expect(/(a)\1{2}0/).to stay_the_same + end +end diff --git a/spec/lib/js_regex/second_pass/expand_bounded_capture_loops_spec.rb b/spec/lib/js_regex/second_pass/expand_bounded_capture_loops_spec.rb new file mode 100644 index 0000000..15d3c57 --- /dev/null +++ b/spec/lib/js_regex/second_pass/expand_bounded_capture_loops_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +describe JsRegex::SecondPass::ExpandBoundedCaptureLoops do + before { stub_const('JsRegex::SecondPass::STEPS', [described_class]) } + + it 'leaves loops without referenced captures unchanged' do + expect(/((a)|b){2,4}/).to stay_the_same + end + + it 'leaves nonempty captures that participate in every iteration unchanged' do + expect(/(a){2,4}\1/).to stay_the_same + end + + it 'leaves unbounded loops to the other steps' do + expect(/(?:(a)|b)+\1/).to stay_the_same + end + + it 'gives each iteration its own capture and records their matching order' do + context = JsRegex::Converter::Context.new(target: ES2009) + tree = JsRegex::Converter.convert(Regexp::Parser.parse(/(?:(a)|b){2}\1/), context) + described_class.call(tree) + + expect(tree.children.first.quantifier).to be_nil + reference = tree.children.last + expect(reference.type).to eq(:capture_backref) + expect(reference.children.map(&:reference)).to eq([2, 1]) + end +end diff --git a/spec/lib/js_regex/second_pass/expand_constant_capture_loops_spec.rb b/spec/lib/js_regex/second_pass/expand_constant_capture_loops_spec.rb new file mode 100644 index 0000000..d774b6a --- /dev/null +++ b/spec/lib/js_regex/second_pass/expand_constant_capture_loops_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +describe JsRegex::SecondPass::ExpandConstantCaptureLoops do + before { stub_const('JsRegex::SecondPass::STEPS', [described_class]) } + + it 'leaves loops without referenced captures unchanged' do + expect(/(?:(a)|b)+/).to stay_the_same + end + + it 'leaves captures with several possible values to the other steps' do + expect(/(?:(a+)|(b))+?\1\2/).to stay_the_same + end + + it 'retains constant captures made on different iterations' do + expect(/\A(?:(a)|(b))+\1\2\z/).to keep_matching('abab', 'aabbab', 'babaab') + .and keep_not_matching('ab', 'abb', 'aba', 'aabb') + end + + it 'uses retained captures in later iterations even when branches overlap' do + expect(/\A(?:(a)|ab\1)+\z/).to keep_matching('a', 'aa', 'aaba') + .and keep_not_matching('ab', 'aba', 'aababa') + end + + it 'rejects a forward reference until an earlier iteration has captured it' do + expect(/\A(?:b\1|(a))+\z/).to keep_matching('a', 'aba', 'aaba') + .and keep_not_matching('ba', 'b', 'ab') + end + + it 'checks retained participation in conditional loop branches that share a prefix' do + expect(/\A(?:(a)|(?(1)ab|c))+\1\z/).to keep_matching('aa', 'aaba', 'caaba') + .and keep_not_matching('c', 'ba', 'aca') + end + + it 'keeps lazy match order when two captures share a prefix' do + expect(/(?:(aa)|(a))+?\1\2/).to keep_capturing( + 'aaaaaa' => 'aaaaaa', 'aaaaaaa' => 'aaaaaa', 'aaaaaaaa' => 'aaaaaaaa' + ) + end + + it 'retains a single constant capture for a conditional after a lazy loop' do + expect(/(?:(a)|ab)+?(?(1)c|a)/).to keep_capturing('aabc' => 'aabc', 'aaba' => 'aba') + expect(/\A(?:(a)|ab)+?(?(1)c|a)\z/).to keep_matching('aabc', 'aba') + .and keep_not_matching('aaba', 'ab', 'a') + expect(/(?:(a)|ab)*?(?(1)c|a)/).to keep_capturing('aabc' => 'a', 'abc' => 'a') + end + + it 'tracks participation without fixing the value of captures used only by conditions' do + expect(/(?:(a+)|ab)+?(?(1)c|a)/).to keep_capturing('aabc' => 'aabc', 'aaba' => 'aba') + expect(/(?:([ab]+)|b)+?(?(1)c|a)/).to keep_capturing('aba' => 'ba', 'aaba' => 'ba') + expect(/(?:(ab+)|ab)+?(?(1)c|a)/).to keep_capturing('ababa' => 'aba') + end + + it 'still rejects variable capture values read by a backreference' do + expect(/(?:(a+)|ab)+?(?:c|\1)/).to stay_the_same + end +end diff --git a/spec/lib/js_regex/second_pass/expand_nullable_captures_spec.rb b/spec/lib/js_regex/second_pass/expand_nullable_captures_spec.rb new file mode 100644 index 0000000..e18ecd6 --- /dev/null +++ b/spec/lib/js_regex/second_pass/expand_nullable_captures_spec.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +describe JsRegex::SecondPass::ExpandNullableCaptures do + before { stub_const('JsRegex::SecondPass::STEPS', [described_class]) } + + it 'leaves required captures unchanged' do + expect(/(a*)b\1/).to stay_the_same + end + + it 'keeps the final empty capture of a lazy repeated character run' do + expect(/(a*)+?\1/).to become('a*()') + .and keep_capturing('' => ['', ''], 'a' => ['a', ''], 'aaab' => ['aaa', '']) + expect(/([ab]*)*?\1/).to become('[ab]*()') + .and keep_capturing('abba' => ['abba', ''], 'c' => ['', '']) + expect(/((?:a)*)+?\1/).to become('(?:a)*()') + .and keep_capturing('aaa' => ['aaa', '']) + expect(/((a*)+?\2)a/).to keep_capturing('aaa' => ['aaa', 'aa', '']) + end + + it 'does not collapse lazy runs when their continuation or body can change the ending' do + [/(a*)+?\1b/, /(a*)+?\1?/, /(a*?)+?\1/, /((?:ab)*)+?\1/].each do |regexp| + expect(regexp).to stay_the_same + end + end + + it 'leaves unreferenced optional captures unchanged' do + expect(/(a*)?b/).to stay_the_same + end + + it 'puts greedy and lazy optional choices in their original order' do + [/\A(a*)?\1/, /\A(a*)??\1/].each do |regexp| + context = JsRegex::Converter::Context.new(target: ES2009) + tree = JsRegex::Converter.convert(Regexp::Parser.parse(regexp), context) + described_class.call(tree) + branches = tree.children[1].children[1].children + first_capture = branches.first.children.first + last_capture = branches.last.children[1] + if regexp.source.include?('??') + expect(first_capture.zero_quantified?).to be(true) + expect(last_capture.quantifier).to be_nil + else + expect(first_capture.quantifier).to be_nil + expect(last_capture.zero_quantified?).to be(true) + end + end + end + + it 'copies the continuation into each alternative that can skip the capture' do + expect(/\A(?:(a*)|b|d)(?:c|\1)\z/) + .to become('^(?:(?:(a*)(?:c|\\1)$)|(?:b)(?:c|(?!))$|(?:d)(?:c|(?!))$)') + end + + it 'copies the continuation into the choices of an optional parent' do + expect(/\A(?:(a*)b)?c(?:d|\1)\z/) + .to become('^(?:(?:(a*)bc(?:d|\\1)$)|(?:()){0}c(?:d|(?!))$)') + end + + it 'selects conditional branches using each optional choice' do + expect(/\A(a*)?(?(1)b|a)\1?/).to become('^(?:(a*)(?:(?:b))\\1?|(){0}(?:(?:a))(?!)?)') + expect(/\A(a*)?(?(1)b)\z/).to keep_matching('', 'b', 'ab', 'aab') + .and keep_not_matching('a') + end + + it 'handles referenced captures nested inside the optional capture' do + expect(/\A((a*))?\1\2\z/).to keep_matching('', 'aaa') + .and keep_not_matching('a', 'aa') + end + + it 'handles empty captures whose participation depends on an assertion' do + expect(/\A((?=a))?a(?:b|\1)\z/).to keep_matching('a', 'ab') + .and keep_not_matching('', 'b') + end + + it 'keeps an empty loop ending outside the repeated nonempty captures' do + context = JsRegex::Converter::Context.new(target: ES2009) + tree = JsRegex::Converter.convert(Regexp::Parser.parse(/\A(a*)*\1\z/), context) + described_class.call(tree) + prefix, ending = tree.children[1].children + expect(prefix.quantifier.to_s).to eq('*') + present_ending = ending.children[1].children.first.children.first + empty_capture = present_ending.children.first + expect(empty_capture.type).to eq(:captured_group) + expect(empty_capture.children).to eq(['(', ')']) + expect(empty_capture.quantifier).to be_nil + end +end diff --git a/spec/lib/js_regex/second_pass/handle_non_participating_backrefs_spec.rb b/spec/lib/js_regex/second_pass/handle_non_participating_backrefs_spec.rb new file mode 100644 index 0000000..60aff0b --- /dev/null +++ b/spec/lib/js_regex/second_pass/handle_non_participating_backrefs_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +describe JsRegex::SecondPass::HandleNonParticipatingBackrefs do + before { stub_const('JsRegex::SecondPass::STEPS', [described_class]) } + + it 'preserves references to completed captures' do + expect(/(a)\1/).to stay_the_same + end + + it 'requires optional captures to participate when their backreferences are used' do + expect(/(a)?b\1/).to keep_matching('aba') + .and keep_not_matching('b', 'bbb') + end + + it 'rejects references to captures that have not finished' do + expect(/\1(a)/).to become('(?!)(a)') + expect(/(a\1)/).to become('(a(?!))') + end + + it 'rejects references to captures in another alternative' do + expect(/(a)|\1/).to become('(a)|(?!)') + .and keep_matching('a') + .and keep_not_matching('', 'b') + end + + it 'preserves references within the same branch and after nested alternatives' do + expect(/(?:(a)\1|b)/).to stay_the_same + expect(/(a)(?:b|c)\1/).to stay_the_same + end + + it 'does not treat participation checks as ordinary backreferences' do + tree = JsRegex::Node.new('\\1', reference: 1, type: :capture_check) + described_class.call(tree) + expect(tree.to_s).to eq('\\1') + end + + it 'allows another branch or an optional backreference to skip the capture' do + expect(/\A(a)?b(?:c|\1)\z/).to keep_matching('bc', 'aba') + .and keep_not_matching('b') + expect(/\A(a)?b\1?\z/).to keep_matching('b', 'ab', 'aba') + end + + it 'requires captures nested in optional groups, alternatives, or negative assertions' do + expect(/\A(?:(a))?b\1\z/).to keep_matching('aba').and keep_not_matching('b') + expect(/\A(?:(a)|b)\1\z/).to keep_matching('aa').and keep_not_matching('b') + expect(/\A(?!(a))b\1\z/).to keep_not_matching('b') + end + + it 'requires nullable captures to participate when their references are mandatory' do + expect(/\A(?:(a*)|b)\1\z/).to keep_matching('', 'aa').and keep_not_matching('b') + expect(/\A(?:(a*)b)?c\1\z/).to keep_matching('bc', 'abca').and keep_not_matching('c') + expect(/\A(a*){0}b\1\z/).to keep_not_matching('b') + end + + it 'keeps enclosing optional groups and unrelated alternatives optional' do + expect(/\A(?:(a*)b\1)?c\z/).to keep_matching('c', 'bc', 'abac') + expect(/\A(?:(?:(a*)|b)\1|c)\z/).to keep_matching('', 'aa', 'c').and keep_not_matching('b') + end + + it 'rejects nullable captures from successful negative assertions' do + expect(/\A(?!(a*)a)b\1\z/).to keep_not_matching('b') + end +end diff --git a/spec/lib/js_regex/second_pass/preserve_repeated_captures_spec.rb b/spec/lib/js_regex/second_pass/preserve_repeated_captures_spec.rb new file mode 100644 index 0000000..e248e2f --- /dev/null +++ b/spec/lib/js_regex/second_pass/preserve_repeated_captures_spec.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +describe JsRegex::SecondPass::PreserveRepeatedCaptures do + before { stub_const('JsRegex::SecondPass::STEPS', [described_class]) } + + it 'leaves repetitions without backreferences unchanged' do + expect(/((a)|b)+/).to stay_the_same + end + + it 'leaves captures that participate in every iteration unchanged' do + expect(/(a)+\1/).to stay_the_same + end + + it 'ends each update cycle with a participating capture' do + context = JsRegex::Converter::Context.new(target: ES2009) + tree = JsRegex::Converter.convert(Regexp::Parser.parse(/(?:(a)|b)+\1/), context) + described_class.call(tree) + + expect(tree.children.last.type).to eq(:backref) + expect(tree.children.last.reference).to eq(1) + expect(tree.to_s).to eq('(?:(?:(?:(?!)|b)*?(?:(a)|b(?!)))+(?:(?!)|b)*|(?:(?!)|b)+)\\1') + end + + it 'marks references to duplicated enclosing captures for history resolution' do + context = JsRegex::Converter::Context.new(target: ES2009) + tree = JsRegex::Converter.convert(Regexp::Parser.parse(/((a)|b)+\2\1/), context) + described_class.call(tree) + + expect(tree.children[-2].reference).to eq(3) + history = tree.children.last + expect(history.type).to eq(:capture_backref) + expect(history.children.map(&:reference)).to eq([5, 4, 2, 1]) + end + + it 'leaves ambiguous lazy loops with optional references unchanged' do + expect(/(?:(a+)|a)+?\1?/).to stay_the_same + end + + it 'retains variable-length captures when alternatives overlap' do + expect(/\A(?:(ab+)|a)+\1\z/).to keep_matching('abaab', 'abbaabb') + end + + it 'recognizes required references within enclosing branches and optional groups' do + expect(/\A(?:(?:(a)|ab)+\1|c)\z/).to keep_matching('aa', 'aaba', 'c') + expect(/\A(?:(?:(a)|ab)+\1)?\z/).to keep_matching('', 'aa', 'aaba') + end + + it 'leaves overlapping loops with several retained captures unchanged' do + expect(/(?:(aa)|(a))+?\1\2/).to stay_the_same + end +end diff --git a/spec/lib/js_regex/second_pass/resolve_capture_references_spec.rb b/spec/lib/js_regex/second_pass/resolve_capture_references_spec.rb new file mode 100644 index 0000000..1005267 --- /dev/null +++ b/spec/lib/js_regex/second_pass/resolve_capture_references_spec.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +describe JsRegex::SecondPass::ResolveCaptureReferences do + before { stub_const('JsRegex::SecondPass::STEPS', [described_class]) } + + describe 'backreferences' do + it 'uses the latest mandatory copy of a capture' do + expect(/([ab])\g<1>\1/).to become('([ab])([ab])\\2') + end + + it 'uses the latest participating copy of an optional capture' do + expect(/\A\g<1>([ab])?\1\z/).to keep_matching('aa', 'bb', 'abb', 'baa') + .and keep_not_matching('a', 'b', 'aba', 'bab') + end + + it 'preserves greedy matching of optional copies' do + expect(/\A\g<1>(a)?\1/).to keep_capturing('aaa' => 'aaa') + end + + it 'rejects references when no copy has finished capturing' do + references = [1, 2].map { |position| JsRegex::Node.new("\\#{position}", reference: position, type: :backref) } + tree = JsRegex::Node.new(*references, type: :capture_backref) + described_class.call(tree) + expect(tree.to_s).to eq('(?!)') + end + end + + describe 'conditions' do + it 'leaves ordinary conditionals for the permutation step' do + tree = JsRegex::Node.new(type: :conditional, reference: 1) + expect(described_class.call(tree)).to equal(tree) + expect(tree.type).to eq(:conditional) + end + + it 'checks all copies of a capture made by subexpression calls' do + expect(/\A(a)?\g<1>(?(1)b|c)\z/).to keep_matching('ab', 'aab') + .and keep_not_matching('ac', 'aac') + end + + it 'uses an earlier capture when a later optional copy is skipped' do + expect(/\A\g<1>(a)?(?(1)b|c)\z/).to keep_matching('ab', 'aab') + .and keep_not_matching('ac', 'aac') + end + + it 'supports conditions without an else branch' do + expect(/\A(a)?\g<1>?(?(1)b)\z/).to keep_matching('', 'ab', 'aab') + .and keep_not_matching('b', 'a', 'aa') + end + + it 'preserves later reference numbers when discarding branch captures' do + expect(/\A(a)\g<1>(?(1)b|(c(d)))(e)\4\z/) + .to keep_matching('aabee') + .and keep_not_matching('aabed', 'aacdee', 'aabe') + end + end + + describe 'duplicate-name references' do + it 'tries mandatory captures with the same name in reverse order' do + expect(/(?a)(?b)\k/).to become('(a)(b)(?:\\2|(?!\\2)\\1)') + end + + it 'requires optional nonempty captures to participate without changing greediness' do + expect(/\A(?a)?(?a)?\k/).to keep_matching('aa', 'aaa') + .and keep_not_matching('', 'a') + .and keep_capturing('aaa' => 'aaa') + end + + it 'marks lazy nullable optional captures for conditional expansion' do + context = JsRegex::Converter::Context.new(target: ES2009) + tree = JsRegex::Converter.convert(Regexp::Parser.parse(/(?a*)(?b*)??\k/), context) + described_class.call(tree) + conditional = nil + visit = lambda do |node| + return unless node.is_a?(JsRegex::Node) + + conditional ||= node if node.type == :conditional + node.children.each { |child| visit.call(child) } + end + visit.call(tree) + expect(conditional).not_to be_nil + expect(conditional.reference).to eq(2) + end + + it 'guards captures in negative assertions' do + expect(/\A(?!(?a))(?b)\k\z/).to keep_matching('bb') + .and keep_not_matching('b', 'a', 'ab') + end + end +end diff --git a/spec/lib/js_regex/second_pass/substitute_root_level_keep_mark_spec.rb b/spec/lib/js_regex/second_pass/substitute_root_level_keep_mark_spec.rb new file mode 100644 index 0000000..85dfb40 --- /dev/null +++ b/spec/lib/js_regex/second_pass/substitute_root_level_keep_mark_spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +describe JsRegex::SecondPass::SubstituteRootLevelKeepMark, targets: [ES2018] do + before { stub_const('JsRegex::SecondPass::STEPS', [described_class]) } + + it 'leaves trees without keep marks unchanged' do + expect(/abc/).to stay_the_same + end + + it 'replaces a keep mark with a lookbehind' do + expect(/a\Kb/).to become('(?<=a)b') + end + + it 'uses the last keep mark in a sequence' do + expect(/a\Kb\Kc/).to become('(?<=ab)c') + .and keep_matching('abc', with_results: ['c']) + end + + it 'keeps root alternatives outside the lookbehind' do + expect(/a\Kb|c\Kd/).to become('(?<=a)b|(?<=c)d') + .and keep_matching('ab', 'cd') + .and keep_not_matching('a', 'c') + end +end diff --git a/spec/lib/js_regex/second_pass_spec.rb b/spec/lib/js_regex/second_pass_spec.rb index ac22878..8047ad3 100644 --- a/spec/lib/js_regex/second_pass_spec.rb +++ b/spec/lib/js_regex/second_pass_spec.rb @@ -1,83 +1,318 @@ # frozen_string_literal: true -require 'spec_helper' - +# Integration tests for the combined second pass. Individual steps are tested +# in the specs under second_pass/. describe JsRegex::SecondPass do - describe '::alternate_conditional_permutations' do - it 'replaces one-branch conditionals with equivalent alternations' do - expect(/-(<)?a(?(1)>)-/).to\ - become(/(?:-(<){0}a(?:(?:>){0})-)|(?:-(<)a(?:(?:>))-)/) - .and keep_matching('-a-', '--') - .and keep_not_matching('--') - end + it 'leaves trees that do not need further processing unchanged' do + expect(/foo/).to stay_the_same + end - it 'keeps 0..(n>1) quantifiers of the target group' do - expect(/-(<){,2}a(?(1)>)-/).to\ - become(/(?:-(<){0}a(?:(?:>){0})-)|(?:-(<){1,2}a(?:(?:>))-)/) - .and keep_matching('--', '-<-') - .and keep_not_matching('--') - end + it 'leaves bounded backreferences without captured groups intact' do + expect(/\A(a)\1{2}0\z/).to become(/^(a)\1{2}0$/) + .and keep_matching('aaa0') + .and keep_not_matching('aa0', 'aaaa0') + end - it 'keeps possessive quantifiers of the target group' do - expect(/-(<)?+a(?(1)>)/).to\ - become(/(?:-(?:(?=((<){0}))\2)a(?:(?:>){0}))|(?:-(?:(?=((<)))\4)a(?:(?:>)))/) - .and keep_matching('-', '-a') - .and keep_not_matching('-)/).to\ + become(/(?:-(?:(?=((){0}))\1)a(?:))|(?:-(?:(?=((<)))\3)a(?:(?:>)))/) + .and keep_matching('-', '-a') + .and keep_not_matching('-|b)-/).to\ - become(/(?:-(<){0}a(?:(?:>){0}(?:b))-)|(?:-(<)a(?:(?:>)(?:b){0})-)/) - .and keep_matching('-ab-', '--') - .and keep_not_matching('-a>-', '--') - end + it 'checks captures made by subexpression calls in conditions' do + expect(/\A(a)?\g<1>(?(1)b|c)\z/).to keep_matching('ab', 'aab') + .and keep_not_matching('ac', 'aac') + expect(/\A(?a)?\g(?()b|c)\z/).to keep_matching('ab', 'aab') + .and keep_not_matching('ac', 'aac') + end - it 'replaces named conditionals with equivalent alternations' do - expect(/-(?<)?a(?()>)-/).to\ - become(/(?:-(<){0}a(?:(?:>){0})-)|(?:-(<)a(?:(?:>))-)/) - .and keep_matching('-a-', '--') - .and keep_not_matching('--') - end + it 'retains earlier participating captures when checking conditions' do + expect(/\A\g<1>(a)?(?(1)b|c)\z/).to keep_matching('ab', 'aab') + .and keep_not_matching('ac', 'aac') + expect(/\A(a)\g<1>?(?(1)b|c)\z/).to keep_matching('ab', 'aab') + .and keep_not_matching('ac', 'aac') + end - it 'replaces quantified conditionals with equivalent alternations' do - expect(/-(<)?a(?(1)>){3}-/).to\ - become(/(?:-(<){0}a(?:(?:>){0}){3}-)|(?:-(<)a(?:(?:>)){3}-)/) - .and keep_matching('-a-', '->>-') - .and keep_not_matching('--') + it 'recognizes participating empty captures made by required calls' do + expect(/\A(a?)?\g<1>(?(1)b|c)\z/).to keep_matching('b', 'ab', 'aab') + .and keep_not_matching('c', 'ac', 'aac') + end + + it 'delimits backreferences after conditional expansion renumbers them' do + expect(/\A(a)(?(1)b)\k<1>0\z/) + .to keep_matching('aba0') + .and keep_not_matching('ab0', "ab\b") + end + + it 'checks nullable captures only on the branches that reference them' do + expect(/\A(?:(a*)|b)(?:c|\1)\z/).to keep_matching('', 'c', 'aa', 'bc') + .and keep_not_matching('b', 'a') + end + + it 'checks nullable captures skipped by an optional parent' do + expect(/\A(?:(a*)b)?c(?:d|\1)\z/).to keep_matching('cd', 'bc', 'abca') + .and keep_not_matching('c', 'abc') + end + + it 'distinguishes skipped from empty duplicate-name captures' do + expect(/\A(?a)?(?b*)?\k\z/) + .to keep_matching('', 'a', 'aa', 'bb', 'abbbb') + .and keep_not_matching('b', 'ab') + end + + # Duplicate-name backreferences were fixed in Ruby 3.1.3: https://bugs.ruby-lang.org/issues/18631 + it 'preserves greedy and lazy choices when duplicate-name candidates can match empty', + if: (Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new('3.1.3')) do + [/\A(?a)?(?a*)?\k/, /\A(?a)?(?a|)?\k/, + /\A(?a*)?(?a*)??\k/, /\A(?a)??(?a*)??\k/, + /\A(?a)??(?a*?)??\k/].each do |regexp| + examples = %w[a aa aaa aaaa].to_h { |str| [str, regexp.match(str)[0]] } + expect(regexp).to keep_capturing(examples) end + end + + it 'preserves greedy choices when a duplicate-name candidate contains another capture' do + expect(/\A(?a)?(?(?a*))?\k/).to keep_capturing('aaa' => 'aaa') + end - it 'replaces successive conditionals with equivalent alternations' do - # resulting source is not speced because it is very long - expect(/(<)?a(?(1)>)(<)?b(?(2)>)/) - .to keep_matching('ab', 'a', 'b', '') - .and keep_not_matching('a>b', 'a)?(?(?=a))?\k/).to keep_capturing('aa' => 'a') + end + + it 'uses the final empty iteration of a repeated duplicate-name candidate' do + expect(/\A(?a)?(?a*)*\k/).to keep_capturing('aaa' => 'aaa') + end + + it 'rejects references to captures skipped by conditional branches' do + expect(/\A(a)?(?(1)b|c)\1\z/).to keep_matching('aba').and keep_not_matching('c') + expect(/\A(a)?(?(1)(b)|c)\2\z/).to keep_matching('abb').and keep_not_matching('c') + end + + it 'requires nullable captures from conditional branches to participate' do + expect(/\A(c)?(?(1)(a*)|b)\2\z/).to keep_matching('c', 'caa').and keep_not_matching('b') + end + + it 'keeps discarded conditional captures unmatched after nullable expansion' do + expect(/\A(a*)?(?(1)b|(c(d)))(?(3)e|f)\z/) + .to keep_matching('bf', 'aabf', 'cde') + .and keep_not_matching('be', 'cdf', 'aabe') + expect(/\A(a*)?(?(1)(b))(c)\3\z/) + .to keep_matching('bcc', 'aabcc', 'cc') + .and keep_not_matching('bc', 'c', 'aacc') + end + + it 'keeps nullable conditional choices after preceding greedy quantifiers' do + expect(/\A(a*)?(?(1)b|a)\1?/).to keep_capturing('aaab' => 'aaab', 'aaa' => 'a') + end + + it 'remaps enclosing captures after splitting a repetition' do + expect(/\A((a)|b)+\2\1\z/).to keep_matching('aaa', 'abab').and keep_not_matching('aa') + expect(/\A((a)|b)+\1\2\z/).to keep_matching('aaa', 'abba').and keep_not_matching('aa') + expect(/\A((a)?)+\1\2\z/).to keep_matching('aaa').and keep_not_matching('') + end + + it 'preserves greedy, lazy, and bounded repetitions with retained captures' do + [/((a)|b)+\2/, /((a)|b)+?\2/, /((a)|b)*\2/, /((a)|b)*?\2/, + /((a)|b){2,4}\2/, /((a)|b){2,}\2/, /((a)|b){2}\2/].each do |regexp| + examples = %w[aa aba abaa baa babaa bbb].map do |input| + [input, regexp.match(input) { |match| match[0] }] + end.to_h + expect(regexp).to keep_capturing(examples) end + end + + it 'retains variable-width captures in bounded repetitions' do + expect(/\A(?:(ab+)|a){1,3}\1\z/).to keep_matching('abaab', 'abbaabb') + .and keep_not_matching('aba') + end + + it 'allows repeated empty iterations when a finite bound prevents an endless loop' do + expect(/\A(a*){0,2}\1\z/).to keep_matching('', 'a', 'aa', 'aaa') + expect(/\A(a*?){0,2}(?(1)b|a)\1?/).to keep_capturing('aba' => 'aba') + end + + it 'retains the last capture when a bounded loop stops after repeated empty iterations' do + expect(/\A(a*?){1,3}(?(1)b|a)\1?/).to keep_capturing('aba' => 'ab', 'aaba' => 'aaba') + end + + it 'retains captures from optional parts of repeated groups' do + expect(/\A(?:(a)?b)+\1\z/).to keep_matching('aba', 'abba', 'babba') + .and keep_not_matching('b', 'bb', 'ab') + end - it 'replaces nested conditionals with equivalent alternations' do - # resulting source is not speced because it is very long - expect(/-(<)?(a)?(b)(?(2)(?(1)->|>>))-/) - .to keep_matching( - '--', # 1: true, 2: true - '->-', # 1: false, 2: true - '--b-', # 1: false, 2: false - ).and keep_not_matching( - '->-', # 1: true, 2: true - '->-', # 1: true, 2: false - '--b>>-', # 1: false, 2: false - ) + it 'allows iterations that skip a retained nullable capture' do + expect(/\A(?:(a*)|b)+\1\z/).to keep_matching('', 'a', 'b', 'ab', 'aba', 'bba') + .and keep_not_matching('c', 'abc') + end + + it 'records a final empty iteration outside a repeated nullable capture' do + expect(/\A(a*)*\1\z/).to keep_matching('', 'a', 'aa', 'aaa') + end + + it 'keeps empty-match preference when a repeated body mixes greedy and lazy parts' do + [[/\A(a*?)*\1a/, 'aaa', 'a'], + [/\A(a*?b*)*\1/, 'b', 'b'], + [/\A(a*?b*)*\1/, 'aa', ''], + [/\A(a*b*?)*\1/, 'a', 'a'], + [/\A(a*b*?)*\1\z/, 'b', 'b']].each do |regexp, input, expected| + expect(regexp).to keep_capturing(input => expected) end + end + + it 'uses the final empty capture when evaluating conditions after a loop' do + expect(/\A(a*?)*(?(1)b|a)\1?/).to keep_capturing('aba' => 'ab') + end + + it 'can backtrack from an empty loop ending to the previous nonempty capture' do + expect(/\A(a*)*(?!\1)a\z/).to keep_matching('a', 'aaa') + .and keep_not_matching('aa') + end + + it 'retains captures through nested repetitions' do + expect(/\A(?:(?:(a)|b)+|c)+\1\z/).to keep_matching('aa', 'aba', 'aca', 'abca') + .and keep_not_matching('b', 'bc', 'c') + end + + it 'retains fixed-width character sets and astral captures' do + expect(/\A(?:([ab])|c)+\1\z/).to keep_matching('aca', 'bcb', 'abcb') + .and keep_not_matching('c', 'acb') + expect(/\A(?:(😋)|b)+\1\z/).to keep_matching('😋b😋').and keep_not_matching('b') + end - it 'adapts backref numbers in the created alternations branches' do - expect(/()(?(1))\1/).to\ - become(/(?:(){0}(?:(?:){0})\1)|(?:()(?:(?:))\2)/) - expect(/(a)(b)(?(1)c)(d)\2/).to\ - become(/(?:(a){0}(b)(?:(?:c){0})(d)\2)|(?:(a)(b)(?:(?:c))(d)\5)/) + it 'checks retained captures when expanding conditionals' do + expect(/\A(?:(a)|b)+(?(1)c|e)\z/).to keep_matching('ac', 'abc', 'be') + .and keep_not_matching('bc') + expect(/\A(d)?((a)|b)+\3(?(1)e|c)\z/).to keep_matching('aac', 'abac', 'daae', 'dabae') + .and keep_not_matching('bc', 'dbe') + end + + it 'retains variable-width captures without changing loop greediness' do + [/\A(?:(a+)|b)+\1/, /\A(?:(a+)|b)+?\1/].each do |regexp| + examples = %w[aba aaba abaa aaaba aaaaa].to_h { |str| [str, regexp.match(str)[0]] } + expect(regexp).to keep_capturing(examples) end + end + + it 'retains variable-length runs when skipped alternatives share their character' do + expect(/(?:(a+)|[ab])+\1/).to keep_capturing('aba' => 'aba', 'aaba' => 'aaba') + end + + it 'retains variable-width captures with several possible characters' do + expect(/\A(?:(ab+)|a)+\1\z/).to keep_matching('abaab', 'abbaabb') + .and keep_not_matching('aba', 'ababb') + expect(/\A(?:([ab]+)|a)+\1\z/).to keep_matching('bab', 'abaab') + .and keep_not_matching('ba') + end - it 'does nothing if passed a tree that does not need further processing' do - expect(/foo/).to stay_the_same + it 'preserves loop order when the skipped branch comes before the capture' do + expect(/(?:a|(a(?:b)?))+\1/).to keep_capturing('abaab' => 'abaab') + end + + it 'retains fixed-width captures when alternatives share characters or prefixes' do + expect(/\A(?:(ab)|ba)+\1\z/).to keep_matching('abab', 'abbaab').and keep_not_matching('abba') + expect(/(?:(a)|ab)+\1/).to keep_capturing('aaba' => 'aaba') + expect(/(?:([ab])|a)+?\1/).to keep_capturing('baab' => 'baa') + end + + it 'retains multiple captures without changing lazy loop order' do + expect(/(?:(aa)|(a))+?\1\2/).to keep_capturing( + 'aaaaaa' => 'aaaaaa', 'aaaaaaa' => 'aaaaaa', 'aaaaaaaa' => 'aaaaaaaa' + ) + end + + it 'retains captures for required references inside a branch or optional parent' do + expect(/\A(?:(?:(a)|ab)+\1|c)\z/).to keep_matching('aa', 'aaba', 'c') + .and keep_not_matching('aab', 'ab') + expect(/\A(?:(?:(a)|ab)+\1)?\z/).to keep_matching('', 'aa', 'aaba') + .and keep_not_matching('aab', 'ab') + end + + it 'uses captures from earlier iterations inside the loop' do + expect(/\A(?:(a)|b\1)+\z/).to keep_matching('a', 'aba', 'aaba', 'ababa') + .and keep_not_matching('b', 'ab', 'ba') + end + + it 'uses earlier captures inside loop branches that share a prefix' do + expect(/\A(?:(a)|ab\1)+\z/).to keep_matching('a', 'aaba') + .and keep_not_matching('ab', 'aba', 'aababa') + end + + it 'evaluates conditional branches using the current iteration state' do + expect(/\A(?:(a)|(?(1)b|c))+\z/).to keep_matching('c', 'ab', 'cab', 'ccab') + .and keep_not_matching('b', 'bb') + expect(/\A(?:(a)|(?(1)b|c))+\1\z/).to keep_matching('aa', 'aba', 'caba') + .and keep_not_matching('ca', 'ba', 'aca') + end + + it 'checks earlier participation in conditional loop branches that share a prefix' do + expect(/\A(?:(a)|(?(1)ab|c))+\1\z/).to keep_matching('aa', 'aaba', 'caaba') + .and keep_not_matching('c', 'ba', 'aca') + end + + it 'stops a nullable alternative loop at the preferred empty iteration' do + expect(/(?:(a*)|b)+\1/).to keep_capturing('b' => '', 'ab' => 'a', 'ba' => '') + end + + it 'retains both variable-width captures in lazy loops' do + expect(/(?:(a+)|(b+))+?\1\2/).to keep_capturing('abab' => 'abab', 'abaab' => 'abaab') + expect(/\A(?:(a+)|(b+))+?\1\2\z/).to keep_matching('abab').and keep_not_matching('ab') + end + + it 'checks loop participation without moving the conditional ahead of the loop' do + expect(/(?:(a)|b)+(?(1)c|a)/).to keep_capturing('bac' => 'bac') + end + + it 'keeps optional capture preference when selecting a conditional branch' do + expect(/(a)?(?(1)b|a)/).to keep_capturing('ab' => 'ab') + expect(/(a)??(?(1)b|a)/).to keep_capturing('ab' => 'a') + end + + it 'tries later lazy iterations before splitting an earlier capture' do + patterns = [/(?:(a+)|(b+))+?\1\2/, /(?:(a+)|(b+))*?\1\2/, + /(?:(a+)|(b+))+?\2\1/, /((a+)|(b+))+?\2\3/] + patterns.each do |regexp| + examples = %w[abbabab baaabab baabaab baabab abaabab baababa aabbaaabbb].to_h do |input| + [input, regexp.match(input) { |match| match[0] }] + end + expect(regexp).to keep_capturing(examples) end end + + it 'distinguishes no iteration from a final empty capture in conditional loops' do + expect(/(?:(a*)|b)+(?(1)c|a)/).to keep_capturing('bac' => 'bac', 'baac' => 'baac') + .and keep_not_matching('a', 'aa', 'ab') + end + + it 'commits to the first matching duplicate-name reference' do + expect(/(?x)(?xx)\kx/).to keep_not_matching('xxxxx') + .and keep_capturing('xxxxxx' => ['xxxxxx', 'x', 'xx']) + expect(/(?x)(?xx)\k\kx/).to keep_not_matching('xxxxxx', 'xxxxxxx') + .and keep_matching('xxxxxxxx') + end + + it 'can try an older named capture when the newer value does not match' do + expect(/(?a)(?b)\k/).to keep_capturing('aba' => ['aba', 'a', 'b'], 'abb' => ['abb', 'a', 'b']) + expect(/(?xx)(?x)??\k/).to keep_capturing('xxxxx' => 'xxxx') + end + + it 'commits to the first matching value among three captures with the same name' do + expect(/(?a)(?aa)(?b)\ka/).to keep_not_matching('aaabaa') + .and keep_capturing('aaabaaa' => ['aaabaaa', 'a', 'aa', 'b'], 'aaabba' => ['aaabba', 'a', 'aa', 'b']) + end + + it 'keeps the final empty capture when a lazy character run is collapsed' do + expect(/((a*)+?\2)\1/).to keep_capturing('aaaa' => ['aaaa', 'aa', '']) + end + + it 'retains participation for conditions after overlapping lazy alternatives' do + expect(/(?:(a)|ab)+?(?(1)c|a)/).to keep_capturing('aabc' => 'aabc', 'aaba' => 'aba') + expect(/(?:(a+)|ab)+?(?(1)c|a)/).to keep_capturing('aabc' => 'aabc', 'aaba' => 'aba') + end + + it 'tries an optional reference before a skipping iteration of an overlapping lazy loop' do + expect(/(?:(a)|ab)+?\1?/).to keep_capturing('ab' => ['a', 'a']) + expect(/(?:(a+)|ab)+?\1?/).to keep_capturing('ab' => ['a', 'a']) + end + end diff --git a/spec/lib/js_regex/target_spec.rb b/spec/lib/js_regex/target_spec.rb index 0feb0f8..8e4ec88 100644 --- a/spec/lib/js_regex/target_spec.rb +++ b/spec/lib/js_regex/target_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'spec_helper' - describe JsRegex::Target do subject { JsRegex::Target } diff --git a/spec/lib/js_regex_spec.rb b/spec/lib/js_regex_spec.rb index 68d62dd..3c9ed82 100644 --- a/spec/lib/js_regex_spec.rb +++ b/spec/lib/js_regex_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'spec_helper' - describe JsRegex do it 'has a semantic version number' do expect(JsRegex::VERSION).to match(/\A\d+\.\d+\.\d+\z/) diff --git a/spec/matchers_spec.rb b/spec/matchers_spec.rb new file mode 100644 index 0000000..c7a9ea4 --- /dev/null +++ b/spec/matchers_spec.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +describe 'keep_capturing' do + it 'checks the first matched text and chains with other matchers' do + expect(/a+/i).to stay_the_same.and keep_capturing('bAAaAba' => 'AAaA') + end + + it 'checks capture groups when an array is expected' do + expect(/(a)?b/).to keep_capturing('ab' => ['ab', 'a'], 'b' => ['b', nil]) + end + + it 'distinguishes empty matches, empty captures, and missing matches' do + expect(/a*/).to keep_capturing('x' => '') + expect(/()/).to keep_capturing('x' => ['', '']) + expect(/a/).to keep_capturing('x' => nil) + end + + it 'passes quotes, backslashes, and line breaks safely to JavaScript' do + input = "'\"\\\n" + expect(/[\s\S]+/).to keep_capturing(input => input) + end + + it 'rejects an incorrect Ruby result even when JavaScript matches the expectation' do + matcher = keep_capturing('a' => nil) + allow(matcher).to receive(:conversions).and_return(ES2009 => JsRegex.new(/b/)) + + expect(matcher.matches?(/a/)).to eq(false) + expect(matcher.failure_message).to include('expected Ruby to capture nil from "a", got "a"') + end + + it 'rejects incorrect JavaScript groups even when the matched text is correct' do + matcher = keep_capturing('a' => ['a', 'a']) + allow(matcher).to receive(:conversions).and_return(ES2009 => JsRegex.new(/a/)) + + expect(matcher.matches?(/(a)/)).to eq(false) + expect(matcher.failure_message).to include('expected JS to capture ["a", "a"] from "a", got ["a"]') + end + + it 'checks later targets and identifies the input and target on failure' do + matcher = keep_capturing('a' => 'a') + allow(matcher).to receive(:conversions).and_return( + ES2009 => JsRegex.new(/a/), ES2015 => JsRegex.new(/b/) + ) + + expect(matcher.matches?(/a/)).to eq(false) + expect(matcher.failure_message).to include('from "a", got nil', 'for ES2015') + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index cf2b965..6191ba7 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -42,7 +42,7 @@ # This method should be used for all JsRegex initializations in syntax specs. def conversions(rb_regex) active_targets.to_h do |target| - output = $js_regex_cache[[rb_regex, target]] ||= + output = $js_regex_cache[[rb_regex, target, JsRegex::SecondPass::STEPS]] ||= JsRegex.new(rb_regex, target: target) [target, output] end @@ -152,7 +152,7 @@ def error_for_options(js_regex, expected) # are not completely identical between Ruby and JS matching calls. # In that case, don't specify expected results and just check that # a valid string does produce a match. - rb_regex =~ string || @msg = "rb did not match `#{string}`" + rb_regex.match?(string) || @msg = "rb did not match `#{string}`" test_in_js(js_regex, string) || @msg = "js did not match `#{string}`" end end @@ -164,13 +164,38 @@ def error_for_options(js_regex, expected) failure_message { @msg + " for #{@target}" } end +# Check the first match: a String checks its text, an Array also checks groups. +RSpec::Matchers.define(:keep_capturing) do |examples| + match do |rb_regex| + conversions(rb_regex).all? do |target, js_regex| + @target = target + examples.all? do |string, expected| + groups = expected.is_a?(Array) + rb_match = rb_regex.match(string) + rb_capture = rb_match && (groups ? rb_match.to_a : rb_match[0]) + js_match = captures_in_js(js_regex, string) + js_capture = js_match && (groups ? js_match : js_match[0]) + + @msg = if rb_capture != expected + "expected Ruby to capture #{expected.inspect} from #{string.inspect}, got #{rb_capture.inspect}" + elsif js_capture != expected + "expected JS to capture #{expected.inspect} from #{string.inspect}, got #{js_capture.inspect}" + end + @msg.nil? + end + end + end + + failure_message { @msg + " for #{@target}" } +end + RSpec::Matchers.define(:keep_not_matching) do |*test_strings| match do |rb_regex| @errors = [] conversions(rb_regex).all? do |target, js_regex| @target = target test_strings.each do |string| - rb_regex =~ string && @errors.push("rb did match `#{string}`") + rb_regex.match?(string) && @errors.push("rb did match `#{string}`") test_in_js(js_regex, string) && @errors.push("js did match `#{string}`") end @errors.empty? @@ -217,6 +242,10 @@ def matches_in_js(js_regex, string) eval_js("'#{js_escape(string)}'.match(#{js_regex}g);").to_a end +def captures_in_js(js_regex, string) + eval_js("#{js_regex}.exec(#{string.to_json})") +end + def test_in_js(js_regex, string) eval_js("#{js_regex}.test('#{js_escape(string)}');") end diff --git a/tasks/build_prop_map.rake b/tasks/build_prop_map.rake index a7d1527..91ff14a 100644 --- a/tasks/build_prop_map.rake +++ b/tasks/build_prop_map.rake @@ -32,9 +32,15 @@ task :build_prop_map do } JS + utf8_matched_chars = + matched_chars + .to_a + .map { |chr| chr.encode('utf-8', invalid: :replace, undef: :replace, replace: '') } + .reject(&:empty?) + if name_in_js.nil? puts "🟠 #{prop.name} (not supported by JS)" - elsif CharacterSet.of_property(prop) != CharacterSet.new(matched_chars) + elsif CharacterSet.of_property(prop) != CharacterSet.new(utf8_matched_chars) puts "🟡 #{prop.name} (different behavior in JS)" else hash[prop] = name_in_js