From 3c137eb119550874b2b3e27d12b733ca67033377 Mon Sep 17 00:00:00 2001
From: Sutou Kouhei <kou@clear-code.com>
Date: Sun, 28 Feb 2021 06:26:40 +0900
Subject: [PATCH] Fix a parser bug that some data may be ignored before DOCTYPE

HackerOne: HO-1104077

For example, "x<?x y" in "x<?x y\n<!--..." is ignored.

Reported by Juho Nurminen. Thanks!!!
---
 lib/rexml/parsers/baseparser.rb           | 15 ++++++++-------
 test/parse/test_processing_instruction.rb | 19 +++++++++++++++++++
 test/parser/test_ultra_light.rb           |  1 -
 3 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/lib/rexml/parsers/baseparser.rb b/lib/rexml/parsers/baseparser.rb
index 935cbd9e..305b1207 100644
--- a/lib/rexml/parsers/baseparser.rb
+++ b/lib/rexml/parsers/baseparser.rb
@@ -195,11 +195,9 @@ def pull_event
         return [ :end_document ] if empty?
         return @stack.shift if @stack.size > 0
         #STDERR.puts @source.encoding
-        @source.read if @source.buffer.size<2
         #STDERR.puts "BUFFER = #{@source.buffer.inspect}"
         if @document_status == nil
-          #@source.consume( /^\s*/um )
-          word = @source.match( /^((?:\s+)|(?:<[^>]*>))/um )
+          word = @source.match( /\A((?:\s+)|(?:<[^>]*>))/um )
           word = word[1] unless word.nil?
           #STDERR.puts "WORD = #{word.inspect}"
           case word
@@ -257,18 +255,16 @@ def pull_event
               @stack << [ :end_doctype ]
             end
             return args
-          when /^\s+/
+          when /\A\s+/
           else
             @document_status = :after_doctype
-            @source.read if @source.buffer.size<2
-            md = @source.match(/\s*/um, true)
             if @source.encoding == "UTF-8"
               @source.buffer.force_encoding(::Encoding::UTF_8)
             end
           end
         end
         if @document_status == :in_doctype
-          md = @source.match(/\s*(.*?>)/um)
+          md = @source.match(/\A\s*(.*?>)/um)
           case md[1]
           when SYSTEMENTITY
             match = @source.match( SYSTEMENTITY, true )[1]
@@ -349,7 +345,11 @@ def pull_event
             return [ :end_doctype ]
           end
         end
+        if @document_status == :after_doctype
+          @source.match(/\A\s*/um, true)
+        end
         begin
+          @source.read if @source.buffer.size<2
           if @source.buffer[0] == ?<
             if @source.buffer[1] == ?/
               @nsstack.shift
@@ -392,6 +392,7 @@ def pull_event
               unless md
                 raise REXML::ParseException.new("malformed XML: missing tag start", @source)
               end
+              @document_status = :in_element
               prefixes = Set.new
               prefixes << md[2] if md[2]
               @nsstack.unshift(curr_ns=Set.new)
diff --git a/test/parse/test_processing_instruction.rb b/test/parse/test_processing_instruction.rb
index a23513fc..f0c0c24e 100644
--- a/test/parse/test_processing_instruction.rb
+++ b/test/parse/test_processing_instruction.rb
@@ -20,6 +20,25 @@ def test_no_name
 <??>
         DETAIL
       end
+
+      def test_garbage_text
+        # TODO: This should be parse error.
+        # Create test/parse/test_document.rb or something and move this to it.
+        doc = parse(<<-XML)
+x<?x y
+<!--?><?x -->?>
+<r/>
+        XML
+        pi = doc.children[1]
+        assert_equal([
+                       "x",
+                       "y\n<!--",
+                     ],
+                     [
+                       pi.target,
+                       pi.content,
+                     ])
+      end
     end
   end
 end
diff --git a/test/parser/test_ultra_light.rb b/test/parser/test_ultra_light.rb
index 8f4a3980..44fd1d1e 100644
--- a/test/parser/test_ultra_light.rb
+++ b/test/parser/test_ultra_light.rb
@@ -16,7 +16,6 @@ def test_entity_declaration
                        nil,
                        [:entitydecl, "name", "value"]
                      ],
-                     [:text, "\n"],
                      [:start_element, :parent, "root", {}],
                      [:text, "\n"],
                    ],