ci: nokogumbo test suite

.github/workflows/ci.yml

@@ -22,9 +22,36 @@ jobs:
       image: flavorjones/nokogiri-test:mri-3.0
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - run: bundle install --local || bundle install
       - run: bundle exec rake rubocop
 
+  gumbo:
+    needs: ["rubocop"]
+    strategy:
+      fail-fast: false
+      matrix:
+        plat: ["ubuntu", "windows", "macos"]
+    runs-on: ${{matrix.plat}}-latest
+    steps:
+      - name: configure git crlf
+        if: matrix.plat == 'windows'
+        run: |
+          git config --system core.autocrlf false
+          git config --system core.eol lf
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - uses: MSP-Greg/setup-ruby-pkgs@v1
+        with:
+          ruby-version: "3.0"
+          apt-get: "ragel"
+          brew: "ragel"
+          mingw: "ragel"
+          bundler-cache: true
+      - run: bundle exec rake gumbo:test
+
   basic:
     needs: ["rubocop"]
     strategy:
@@ -37,6 +64,8 @@ jobs:
       image: flavorjones/nokogiri-test:${{matrix.image}}
     steps:
       - uses: actions/checkout@v1 # v1 because of https://github.com/actions/checkout/issues/334
+        with:
+          submodules: true
       - run: bundle install --local || bundle install
       - run: bundle exec rake compile -- --${{matrix.sys}}-system-libraries
       - run: bundle exec rake test
@@ -53,6 +82,8 @@ jobs:
       image: flavorjones/nokogiri-test:mri-${{matrix.ruby}}
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: actions/cache@v2
         if: matrix.sys == 'disable'
         with:
@@ -74,6 +105,8 @@ jobs:
       image: flavorjones/nokogiri-test:mri-${{matrix.ruby}}
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: actions/cache@v2
         if: matrix.sys == 'disable'
         with:
@@ -94,6 +127,8 @@ jobs:
       image: flavorjones/nokogiri-test:alpine
     steps:
       - uses: actions/checkout@v1 # v1 because of https://github.com/actions/checkout/issues/334
+        with:
+          submodules: true
       # skip cache because of https://github.com/actions/checkout/issues/334
       - run: bundle install --local || bundle install
       - run: bundle exec rake compile -- --${{matrix.sys}}-system-libraries
@@ -110,6 +145,8 @@ jobs:
       image: flavorjones/nokogiri-test:alpine
     steps:
       - uses: actions/checkout@v1 # v1 because of https://github.com/actions/checkout/issues/334
+        with:
+          submodules: true
       # skip cache because of https://github.com/actions/checkout/issues/334
       - run: bundle install --local || bundle install
       - run: bundle exec rake compile -- --${{matrix.sys}}-system-libraries
@@ -129,6 +166,8 @@ jobs:
       image: flavorjones/nokogiri-test:mri-${{matrix.ruby}}
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: actions/cache@v2
         if: matrix.sys == 'disable'
         with:
@@ -152,6 +191,8 @@ jobs:
       image: flavorjones/nokogiri-test:mri-${{matrix.ruby}}
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: actions/cache@v2
         if: matrix.sys == 'disable'
         with:
@@ -171,6 +212,8 @@ jobs:
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{matrix.ruby}}
@@ -197,6 +240,8 @@ jobs:
           git config --system core.autocrlf false
           git config --system core.eol lf
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: MSP-Greg/setup-ruby-pkgs@v1
         with:
           ruby-version: "${{matrix.ruby}}"
@@ -219,6 +264,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{matrix.ruby}}

.github/workflows/gem-install.yml

@@ -23,6 +23,8 @@ jobs:
       image: "larskanis/rake-compiler-dock-mri-x86_64-linux:1.1.0"
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: actions/cache@v2
         with:
           path: ports/archives
@@ -44,6 +46,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: MSP-Greg/setup-ruby-pkgs@v1
         with:
           ruby-version: "${{matrix.ruby}}"
@@ -63,6 +67,8 @@ jobs:
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: "3.0"
@@ -81,6 +87,8 @@ jobs:
     runs-on: windows-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: MSP-Greg/setup-ruby-pkgs@v1
         with:
           ruby-version: "3.0"
@@ -106,6 +114,8 @@ jobs:
       image: "larskanis/rake-compiler-dock-mri-${{matrix.plat}}:1.1.0"
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: actions/cache@v2
         with:
           path: ports/archives
@@ -126,6 +136,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: "${{matrix.ruby}}"
@@ -144,6 +156,8 @@ jobs:
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: "${{matrix.ruby}}"
@@ -181,6 +195,8 @@ jobs:
       image: "larskanis/rake-compiler-dock-jruby:1.1.0"
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - run: ./scripts/test-gem-build gems java
       - uses: actions/upload-artifact@v2
         with:
@@ -197,6 +213,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: "${{matrix.ruby}}"

.github/workflows/truffle.yml

@@ -22,6 +22,8 @@ jobs:
       image: flavorjones/nokogiri-test:truffle-nightly
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: actions/cache@v2
         with:
           path: ports/archives

.github/workflows/upstream.yml

@@ -24,6 +24,8 @@ jobs:
       image: flavorjones/nokogiri-test:mri-3.0
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - name: Setup libxml2
         run: |
           git clone --depth=1 https://gitlab.gnome.org/GNOME/libxml2
@@ -47,6 +49,8 @@ jobs:
       image: flavorjones/nokogiri-test:mri-3.0
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - name: Setup libxml2
         run: |
           git clone --depth=1 https://gitlab.gnome.org/GNOME/libxml2
@@ -77,6 +81,8 @@ jobs:
           git config --system core.autocrlf false
           git config --system core.eol lf
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: MSP-Greg/setup-ruby-pkgs@ucrt
         with:
           ruby-version: "head"
@@ -101,6 +107,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: MSP-Greg/setup-ruby-pkgs@v1
         with:
           ruby-version: "head"
@@ -118,6 +126,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: true
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: "jruby-head"

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "test/html5lib-tests"]
+	path = test/html5lib-tests
+	url = https://github.com/stevecheckoway/html5lib-tests.git
+	branch = all-error-fixes

CONTRIBUTING.md

@@ -153,6 +153,21 @@ If you want to build Nokogiri against a modified version of libxml2, clone libxm
 That script also takes an optional command to run with the proper environment variables set to use the local libxml2 library, which can be useful when trying to `git bisect` against libxml2.
 
 
+### gumbo HTML5 parser
+
+To run the test suite for the gumbo parser:
+
+``` sh
+bundle exec rake gumbo
+```
+
+To make sure to run additional html5lib tests for Nokogiri's HTML5 parser:
+
+``` sh
+git submodule update --init # test/html5lib-tests
+bundle exec rake compile test
+```
+
 
 ## Style Guide
 
@@ -182,6 +197,7 @@ The `ci.yml` pipeline includes jobs to:
 
 - basic security sanity check: run rubocop
 - fast feedback for obvious failures: run against system libraries on vanilla ubuntu
+- run the gumbo parser tests on ubuntu, macos, and windows
 - run on all supported versions of CRuby:
     - once with packaged libraries
     - once with system libraries
@@ -217,6 +233,13 @@ The `truffle.yml` pipeline tests TruffleRuby nightlies with a few different comp
 We rely heavily on Valgrind to catch memory bugs by running in combination with every version of CRuby. We use suppressions, too -- because some Rubies seem to have memory issues? See the files in the `/suppressions` directory and `/rakelib/test.rake` for more information.
 
 
+### Conventions
+
+- Always checkout the source code including submodules (for the html5lib tests)
+- When testing packaged libraries (not system libraries), cache either `ports/` (for compiled libraries) or `ports/archives/` (for just tarballs)
+  - note that `libgumbo` is built outside of `ports/` to allow us to do this caching safely
+
+
 ## Building gems
 
 As a prerequisite please make sure you have `docker` correctly installed.

Rakefile

@@ -6,4 +6,4 @@
 require "bundler"
 NOKOGIRI_SPEC = Bundler.load_gemspec("nokogiri.gemspec")
 
-task default: [:rubocop, :compile, :test]
+task default: [:rubocop, :gumbo, :compile, :test]

appveyor.yml

@@ -22,6 +22,7 @@ install:
   - ruby --version
   - gem --version
   - gem install bundler --conservative
+  - git submodule update --init
   - bundle config --local path vendor/bundle
   - bundle install
   - IF DEFINED INSTALL_PACKAGES ( ridk exec pacman --noconfirm --needed --sync %INSTALL_PACKAGES% )

gumbo-parser/.gitignore

@@ -1,2 +1,3 @@
 build
+googletest
 src/*.o

rakelib/check-manifest.rake

@@ -33,6 +33,7 @@ task :check_manifest do
     .cross_rubies
     .editorconfig
     .gitignore
+    .gitmodules
     .yardopts
     CHANGELOG.md
     CODE_OF_CONDUCT.md