Add support for publishing maven-metadata.xml

MODULE.bazel

@@ -66,6 +66,7 @@ maven.install(
         "org.apache.maven:maven-core:%s" % _MAVEN_VERSION,
         "org.apache.maven:maven-model:%s" % _MAVEN_VERSION,
         "org.apache.maven:maven-model-builder:%s" % _MAVEN_VERSION,
+        "org.apache.maven:maven-repository-metadata:%s" % _MAVEN_VERSION,
         "org.apache.maven:maven-settings:%s" % _MAVEN_VERSION,
         "org.apache.maven:maven-settings-builder:%s" % _MAVEN_VERSION,
         "org.apache.maven:maven-resolver-provider:%s" % _MAVEN_VERSION,
@@ -86,6 +87,7 @@ maven.install(
         "software.amazon.awssdk:s3:2.26.12",
         "org.bouncycastle:bcprov-jdk15on:1.68",
         "org.bouncycastle:bcpg-jdk15on:1.68",
+        "com.google.http-client:google-http-client:1.45.0",
     ],
     fail_if_repin_required = True,
     fetch_sources = True,

private/rules/java_export.bzl

@@ -15,6 +15,7 @@ def java_export(
         tags = [],
         testonly = None,
         classifier_artifacts = {},
+        publish_maven_metadata = False,
         **kwargs):
     """Extends `java_library` to allow maven artifacts to be uploaded.
 
@@ -77,6 +78,7 @@ def java_export(
         `tags = ["no-javadoc"]`).
       visibility: The visibility of the target
       kwargs: These are passed to [`java_library`](https://bazel.build/reference/be/java#java_library),
+      publish_maven_metadata: Whether to publish a maven-metadata.xml
         and so may contain any valid parameter for that rule.
     """
 
@@ -111,6 +113,7 @@ def java_export(
         testonly = testonly,
         javadocopts = javadocopts,
         classifier_artifacts = classifier_artifacts,
+        publish_maven_metadata = publish_maven_metadata,
         doc_deps = doc_deps,
         doc_url = doc_url,
         doc_resources = doc_resources,
@@ -134,6 +137,7 @@ def maven_export(
         doc_deps = [],
         doc_url = "",
         doc_resources = [],
+        publish_maven_metadata = False,
         toolchains = None):
     """
     All arguments are the same as java_export with the addition of:
@@ -194,6 +198,7 @@ def maven_export(
         `tags = ["no-javadoc"]`).
       doc_resources: Resources to be included in the javadoc jar.
       visibility: The visibility of the target
+      publish_maven_metadata: Whether to publish a maven-metadata.xml
       kwargs: These are passed to [`java_library`](https://bazel.build/reference/be/java#java_library),
         and so may contain any valid parameter for that rule.
     """
@@ -293,6 +298,7 @@ def maven_export(
         tags = tags,
         testonly = testonly,
         toolchains = toolchains,
+        publish_maven_metadata = publish_maven_metadata,
     )
 
     # We may want to aggregate several `java_export` targets into a single Maven BOM POM

private/rules/maven_publish.bzl

@@ -18,7 +18,7 @@ export USE_IN_MEMORY_PGP_KEYS="${{USE_IN_MEMORY_PGP_KEYS:-{use_in_memory_pgp_key
 export PGP_SIGNING_KEY="${{PGP_SIGNING_KEY:-{pgp_signing_key}}}"
 export PGP_SIGNING_PWD="${{PGP_SIGNING_PWD:-{pgp_signing_pwd}}}"
 echo Uploading "{coordinates}" to "${{MAVEN_REPO}}"
-{uploader} "{coordinates}" '{pom}' '{artifact}' '{classifier_artifacts}' $@
+{uploader} "{coordinates}" '{pom}' '{artifact}' '{publish_maven_metadata}' '{classifier_artifacts}' $@
 """
 
 def _escape_arg(str):
@@ -68,6 +68,7 @@ def _maven_publish_impl(ctx):
             pom = ctx.file.pom.short_path,
             artifact = artifacts_short_path,
             classifier_artifacts = ",".join(["{}={}".format(classifier, file.short_path) for (classifier, file) in classifier_artifacts_dict.items()]),
+            publish_maven_metadata = ctx.attr.publish_maven_metadata,
         ),
     )
 
@@ -125,6 +126,10 @@ When signing with GPG, the current default key is used.
             allow_single_file = True,
         ),
         "classifier_artifacts": attr.label_keyed_string_dict(allow_files = True),
+        "publish_maven_metadata": attr.bool(
+            default = False,
+            doc = "Whether to publish a maven-metadata.xml to the Maven repository",
+        ),
         "_uploader": attr.label(
             executable = True,
             cfg = "exec",

private/tools/java/com/github/bazelbuild/rules_jvm_external/maven/BUILD

@@ -42,6 +42,14 @@ java_binary(
             "org.bouncycastle:bcpg-jdk15on",
             repository_name = "rules_jvm_external_deps",
         ),
+        artifact(
+            "com.google.http-client:google-http-client",
+            repository_name = "rules_jvm_external_deps",
+        ),
+        artifact(
+            "org.apache.maven:maven-repository-metadata",
+            repository_name = "rules_jvm_external_deps",
+        ),
     ],
 )
 
@@ -57,3 +65,20 @@ java_binary(
         ),
     ],
 )
+
+java_test(
+    name = "MavenPublisherTest",
+    srcs = [
+        "MavenPublisherTest.java",
+    ],
+    deps = [
+        artifact(
+            "org.apache.maven:maven-repository-metadata",
+            repository_name = "rules_jvm_external_deps",
+        ),
+        artifact(
+            "junit:junit",
+            repository_name = "regression_testing_coursier",
+        ),
+    ]
+)

private/tools/java/com/github/bazelbuild/rules_jvm_external/maven/MavenPublisher.java

@@ -31,6 +31,7 @@
 import com.google.cloud.storage.Storage;
 import com.google.cloud.storage.StorageOptions;
 import com.google.common.base.Splitter;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -57,8 +58,33 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeoutException;
 import java.util.logging.Logger;
+
+import com.google.common.io.CharStreams;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.HttpRequest;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.GenericUrl;
+
+import java.io.UncheckedIOException;
+
+import org.apache.maven.artifact.repository.metadata.Metadata;
+import org.apache.maven.artifact.repository.metadata.Versioning;
+import org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Reader;
+import org.apache.maven.artifact.repository.metadata.io.xpp3.MetadataXpp3Writer;
+
+import java.nio.charset.StandardCharsets;
+import java.net.URISyntaxException;
+import java.util.Optional;
+import java.io.StringReader;
+import java.io.ByteArrayOutputStream;
+import java.util.stream.Collectors;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.io.InputStreamReader;
 
 public class MavenPublisher {
 
@@ -108,8 +134,10 @@ public static void main(String[] args)
         futures.add(upload(repo, credentials, coords, "." + ext, mainArtifact, signingMetadata));
       }
 
-      if (args.length > 3 && !args[3].isEmpty()) {
-        List<String> extraArtifactTuples = Splitter.onPattern(",").splitToList(args[3]);
+      boolean publishMavenMetadata = Boolean.valueOf(args[3]);
+
+      if (args.length > 4 && !args[4].isEmpty()) {
+        List<String> extraArtifactTuples = Splitter.onPattern(",").splitToList(args[4]);
         for (String artifactTuple : extraArtifactTuples) {
           String[] splits = artifactTuple.split("=");
           String classifier = splits[0];
@@ -128,6 +156,16 @@ public static void main(String[] args)
 
       CompletableFuture<Void> all =
           CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
+
+      // uploading the maven-metadata.xml signals to cut over to the new version, so it must be at the end.
+      if (publishMavenMetadata) {
+        all = all.thenCompose(Void -> uploadMavenMetadata(
+                repo,
+                credentials,
+                coords
+       ));
+      }
+
       all.get(30, MINUTES);
     } finally {
       EXECUTOR.shutdown();
@@ -150,6 +188,92 @@ private static boolean isSchemeSupported(String repo) {
     return false;
   }
 
+  /**
+   * Download the pre-existing maven-metadata.xml file if it exists.
+   * If no such file exists, create a default Metadata with the Coordinates provided.
+   */
+  private static CompletableFuture<Metadata> downloadExistingMavenMetadata(
+          String repo,
+          Credentials credentials,
+          Coordinates coords
+  ) {
+    String mavenMetadataUrl =
+            String.format(
+                    "%s/%s/%s/maven-metadata.xml",
+                    repo.replaceAll("/$", ""),
+                    coords.groupId.replace('.', '/'),
+                    coords.artifactId);
+
+    return download(mavenMetadataUrl, credentials).thenApply( optionalFileContents -> {
+      try {
+        if (optionalFileContents.isEmpty()) {
+          // no file so just upload a new one
+          // we must bootstrap
+          Metadata metadata = new Metadata();
+          metadata.setGroupId(coords.groupId);
+          metadata.setArtifactId(coords.artifactId);
+          metadata.setVersioning(new Versioning());
+          return metadata;
+        }
+          return new MetadataXpp3Reader().read(new StringReader(optionalFileContents.get()), false);
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+
+    });
+  }
+
+  /**
+   * Upload the new maven-metadata.xml with the new version included in the version list &
+   * set the latest and release tags in the Metadata XML object.
+   * This function will first download the pre-existing metadata-xml and augment.
+   * If no maven-metadata.xml exists, a new one will be hydrated.
+   */
+  private static CompletableFuture<Void> uploadMavenMetadata(
+          String repo,
+          Credentials credentials,
+          Coordinates coords
+  ) {
+
+    String mavenMetadataUrl =
+            String.format(
+                    "%s/%s/%s/maven-metadata.xml",
+                    repo.replaceAll("/$", ""),
+                    coords.groupId.replace('.', '/'),
+                    coords.artifactId);
+    return downloadExistingMavenMetadata(repo, credentials, coords).thenCompose(
+            metadata -> {
+              try {
+
+                // There is a chance versioning is null; handle it by creating the empty object.
+                Versioning versioning = Optional.ofNullable(metadata.getVersioning()).orElse(new Versioning());
+                versioning.setLatest(coords.version);
+                versioning.setRelease(coords.version);
+                // This may be needed for SNAPSHOT support
+                String timestamp = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
+                versioning.setLastUpdated("20200731090423");
+                versioning.getVersions()
+                        .add(coords.version);
+                // Let's handle adding multiple versions many times by turning it back to a set
+                versioning.setVersions(
+                        versioning.getVersions()
+                                .stream()
+                                .distinct()
+                                .collect(Collectors.toList())
+                );
+                metadata.setVersioning(versioning);
+
+                Path newMavenMetadataXml = Files.createTempFile("maven-metadata", ".xml");
+                ByteArrayOutputStream os = new ByteArrayOutputStream();
+                new MetadataXpp3Writer().write(os, metadata);
+                Files.write(newMavenMetadataXml, os.toByteArray());
+                return upload(mavenMetadataUrl, credentials, newMavenMetadataXml);
+              } catch (Exception e) {
+                throw new RuntimeException(e);
+              }
+    });
+  }
+
   private static CompletableFuture<Void> upload(
       String repo,
       Credentials credentials,
@@ -261,6 +385,60 @@ private static String toHexS(String fmt, String algorithm, byte[] toHash) {
     }
   }
 
+  /**
+   * Attempts to download the file at the given targetUrl.
+   * Valid protocols are: http(s) & file at the moment.
+   */
+  private static CompletableFuture<Optional<String>> download(
+          String targetUrl, Credentials credentials
+  ) {
+    if (targetUrl.startsWith("http")) {
+      return CompletableFuture.supplyAsync( () -> {
+          try {
+            HttpTransport httpTransport = new NetHttpTransport();
+            HttpRequest request = httpTransport.createRequestFactory(initRequest -> {
+              Map<String, List<String>> authHeaders = credentials.getRequestMetadata();
+              // Add the authorization headers to the HTTP request
+              if (authHeaders != null) {
+                for (Map.Entry<String, List<String>> entry : authHeaders.entrySet()) {
+                  initRequest.getHeaders().put(entry.getKey(), entry.getValue());
+                }
+              }
+            }).buildGetRequest(new GenericUrl(targetUrl));
+            HttpResponse response = request.execute();
+            return Optional.of(
+                    CharStreams.toString(new InputStreamReader(response.getContent(), StandardCharsets.UTF_8))
+            );
+          }
+          catch (HttpResponseException e) {
+            if (e.getStatusCode() == 404) {
+              return Optional.empty();
+            } else {
+              throw new UncheckedIOException(e);
+            }
+          } catch (IOException e) {
+            throw new UncheckedIOException(e);
+          }
+      });
+    } else if (targetUrl.startsWith("file://")) {
+      return CompletableFuture.supplyAsync( () -> {
+        try {
+          Path path = Paths.get(new URL(targetUrl).toURI());
+          if (!Files.exists(path)) {
+            return Optional.empty();
+          }
+          return Optional.of(com.google.common.io.Files.asCharSource(path.toFile(), StandardCharsets.UTF_8).read());
+        } catch (IOException e) {
+          throw new UncheckedIOException(e);
+        } catch (URISyntaxException e) {
+          throw new RuntimeException(e);
+        }
+      });
+    } else {
+      throw new IllegalArgumentException("Unsupported protocol for download.");
+    }
+  }
+
   private static CompletableFuture<Void> upload(
       String targetUrl, Credentials credentials, Path toUpload) {
     Callable<Void> callable;