diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc
index 31ebaf20b..c3359ece2 100644
--- a/CHANGELOG.adoc
+++ b/CHANGELOG.adoc
@@ -48,6 +48,7 @@ Build Improvement::
 * Run tests on Java 19 (#1563)
 * Fix upstream tests forcing SNAPSHOT on Asciidoctor gem installation (#1123) (@abelsromero)
 * Fix upstream build removing the explicit plugin repository (#1131)
+* Fix Asciidoctor upstream CI tests (#1178) (@abelsromero)
 
 == 2.5.4 (2022-06-30)
 
diff --git a/build.gradle b/build.gradle
index 322d9fe23..f91c0ff2c 100644
--- a/build.gradle
+++ b/build.gradle
@@ -26,7 +26,7 @@ plugins {
   id "io.sdkman.vendors" version "2.0.0"
   id "signing"
   id "io.github.gradle-nexus.publish-plugin" version "1.1.0"
-  id 'com.github.jruby-gradle.base' version '2.0.0'
+  id 'com.github.jruby-gradle.base' version '2.0.1'
   id 'codenarc'
 }
 
diff --git a/ci/asciidoctor-gem-installer.pom b/ci/asciidoctor-gem-installer.pom
index 3c7184bd2..88305087d 100644
--- a/ci/asciidoctor-gem-installer.pom
+++ b/ci/asciidoctor-gem-installer.pom
@@ -5,10 +5,9 @@
     <groupId>rubygems</groupId>
     <artifactId>asciidoctor</artifactId>
     <version></version>
-    <packaging>gem</packaging>
     <name>Asciidoctor RubyGem Installer</name>
     <description>Installs the upstream Asciidoctor RubyGem to the local repository.</description>
-    <url>http://asciidoctor.org</url>
+    <url>https://asciidoctor.org</url>
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     </properties>
@@ -17,8 +16,7 @@
             <plugin>
                 <groupId>de.saumya.mojo</groupId>
                 <artifactId>gem-maven-plugin</artifactId>
-                <version>1.1.8</version>
-                <extensions>true</extensions>
+                <version>2.0.1</version>
             </plugin>
         </plugins>
     </build>
diff --git a/ci/asciidoctor-pdf-gem-installer.pom b/ci/asciidoctor-pdf-gem-installer.pom
deleted file mode 100644
index b77abef24..000000000
--- a/ci/asciidoctor-pdf-gem-installer.pom
+++ /dev/null
@@ -1,25 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <groupId>rubygems</groupId>
-    <artifactId>asciidoctor-pdf</artifactId>
-    <version></version>
-    <packaging>gem</packaging>
-    <name>Asciidoctor PDF RubyGem Installer</name>
-    <description>Installs the upstream Asciidoctor PDF RubyGem to the local repository.</description>
-    <url>http://asciidoctor.org</url>
-    <properties>
-        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    </properties>
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>de.saumya.mojo</groupId>
-                <artifactId>gem-maven-plugin</artifactId>
-                <version>1.1.8</version>
-                <extensions>true</extensions>
-            </plugin>
-        </plugins>
-    </build>
-</project>
diff --git a/ci/test-asciidoctor-upstream-fix.sh b/ci/test-asciidoctor-upstream-fix.sh
new file mode 100755
index 000000000..aaed11f92
--- /dev/null
+++ b/ci/test-asciidoctor-upstream-fix.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+# This script runs the AsciidoctorJ tests against the specified ref of the Asciidoctor Ruby gem.
+
+set -euo pipefail
+
+# to build against a tag, set REF to a git tag name (e.g., refs/tags/v1.5.2)
+REF=${1:-refs/heads/main}
+if [[ $REF != refs/* ]]; then
+  REF=refs/heads/$REF
+fi
+readonly BASE_REF="${REF##*/}"
+readonly GEMS_INSTALL_PATH="asciidoctorj-core/build/.gems/gems"
+readonly SRC_DIR="asciidoctor-${BASE_REF#v}"
+# readonly WORKING_DIR="build/maven"
+
+rm -rf $WORKING_DIR && mkdir -p $WORKING_DIR
+# pushd $WORKING_DIR
+
+download_gem(){
+}
+
+install_gem(){
+  ./gradlew :asciidoctorj:jrubyPrepare
+
+  wget --quiet -O "${SRC_DIR}.zip" "https://github.com/asciidoctor/asciidoctor/archive/${REF}.zip"
+  unzip -q "${SRC_DIR}.zip" -d "$GEMS_INSTALL_PATH"
+
+  ASCIIDOCTOR_UPSTREAM_VERSION="$(grep 'VERSION' "$GEMS_INSTALL_PATH/$SRC_DIR/lib/asciidoctor/version.rb" | sed "s/.*'\(.*\)'.*/\1/" | sed "s/[.]dev$/.dev-SNAPSHOT/")-SNAPSHOT"
+  mv "$GEMS_INSTALL_PATH/$SRC_DIR" "$GEMS_INSTALL_PATH/asciidoctor-$ASCIIDOCTOR_UPSTREAM_VERSION"
+  # DELETE previous release to ensure nothing leaks
+  ASCIIDOCTOR_CURRENT_VERSION=$(./gradlew :asciidoctorj:properties | grep asciidoctorGemVersion | sed "s/.*:\s*\(.*\)\s*/\1/")
+  rm -rf "$GEMS_INSTALL_PATH/asciidoctor-$ASCIIDOCTOR_CURRENT_VERSION"
+}
+
+install_gem
+
+./gradlew --no-daemon -S -Pskip.signing -PasciidoctorGemVersion="$ASCIIDOCTOR_UPSTREAM_VERSION" check
+
+cp ../../ci/asciidoctor-gem-installer.pom $SRC_DIR/pom.xml
+cd $SRC_DIR
+# force SNAPSHOT trailing to keep always consistency, gem-maven-plugin adds it depending on version format, breaking the build
+readonly ASCIIDOCTOR_VERSION="$(grep 'VERSION' ./lib/asciidoctor/version.rb | sed "s/.*'\(.*\)'.*/\1/" | sed "s/[.]dev$/.dev-SNAPSHOT/")-SNAPSHOT"
+# we don't use sed -i here for compatibility with OSX
+sed "s;<version></version>;<version>$ASCIIDOCTOR_VERSION</version>;" pom.xml > pom.xml.sedtmp && \
+  mv -f pom.xml.sedtmp pom.xml
+
+# we override the jruby version here with one supported by java9, additionally java9 needs some add-opens arguments that need to be ignored on older jvms
+mvn install -B --no-transfer-progress -Dgemspec="asciidoctor.gemspec" -Djruby.version=9.2.17.0 -Djruby.jvmargs="-XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.security.cert=ALL-UNNAMED --add-opens=java.base/java.security=ALL-UNNAMED --add-opens=java.base/java.util.zip=ALL-UNNAMED"
+popd
+
+## Test against installed gem
+./gradlew --no-daemon -S -Pskip.signing -PasciidoctorGemVersion="$ASCIIDOCTOR_VERSION" -PuseMavenLocal=true check
+exit $?
diff --git a/ci/test-asciidoctor-upstream.sh b/ci/test-asciidoctor-upstream.sh
index 4e0651675..6b487378e 100755
--- a/ci/test-asciidoctor-upstream.sh
+++ b/ci/test-asciidoctor-upstream.sh
@@ -27,7 +27,7 @@ sed "s;<version></version>;<version>$ASCIIDOCTOR_VERSION</version>;" pom.xml > p
   mv -f pom.xml.sedtmp pom.xml
 
 # we override the jruby version here with one supported by java9, additionally java9 needs some add-opens arguments that need to be ignored on older jvms
-mvn install -B --no-transfer-progress -Dgemspec="asciidoctor.gemspec" -Djruby.version=9.2.17.0 -Djruby.jvmargs="-XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.security.cert=ALL-UNNAMED --add-opens=java.base/java.security=ALL-UNNAMED --add-opens=java.base/java.util.zip=ALL-UNNAMED"
+mvn install -B --no-transfer-progress -Dgemspec="asciidoctor.gemspec"
 popd
 
 ## Test against installed gem