Skip to content

Commit

Permalink
Generate pseudo-compile jars for bazel java compilation (#518)
Browse files Browse the repository at this point in the history 
Normally, bazel uses `ijar` to prepare a jar containing just
ABI-entries for classfiles. These are stable when non-API breaking
changes are made, and so allow bazel to compile code faster, and are
known as "compile jars"

Because of bazelbuild/bazel#4549
`rules_jvm_external` doesn't use `ijar`, and instead uses the
downloaded jar as the "compile jar". Normally, this is fine, but Bazel
4.0.0 now sets `-Xlint:path` for javac invocations, and this means
that if there's a `Class-Path` manifest entry in a compile jar, the
jars within _that_ will be checked. This can lead to noisy builds:
bazelbuild/bazel#12968

This change generates a "pseudo-compile jar" for `jvm_import`
targets. All it does is repack the input jar, excluding the
`META-INF/MANIFEST.MF` file. This means that we should avoid
compilation problems whilst still working well with Kotlin projects.
  • Loading branch information
@shs96c
shs96c authored Mar 1, 2021
1 parent fe3f5e5 commit ec2c561
Show file tree
Hide file tree
Showing 5 changed files with 267 additions and 110 deletions.
50 changes: 29 additions & 21 deletions private/rules/jvm_import.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,35 @@ def _jvm_import_impl(ctx):

injar = ctx.files.jars[0]
outjar = ctx.actions.declare_file("processed_" + injar.basename, sibling = injar)
ctx.actions.run_shell(
inputs = [injar] + ctx.files._add_jar_manifest_entry,
args = ctx.actions.args()
args.add_all(["--source", injar, "--output", outjar])
args.add_all(["--manifest-entry", "Target-Label:{target_label}".format(target_label = ctx.label)])
ctx.actions.run(
executable = ctx.executable._add_jar_manifest_entry,
arguments = [args],
inputs = [injar],
outputs = [outjar],
command = "\n".join([
# If the jar is signed do not modify the manifest because it will
# make the signature invalid.
"if unzip -l {injar} | grep -qE 'META-INF/.*\\.SF'; then".format(injar = injar.path),
" cp {injar} {outjar}".format(injar = injar.path, outjar = outjar.path),
"else",
" set -e",
" {add_jar_manifest_entry} --source {injar} --manifest-entry 'Target-Label:{target_label}' --output {outjar}".format(
add_jar_manifest_entry = ctx.attr._add_jar_manifest_entry.files_to_run.executable.path,
injar = injar.path,
target_label = ctx.label,
outjar = outjar.path,
),
"fi",
]),
mnemonic = "StampJarManifest",
progress_message = "Stamping the manifest of %s" % ctx.label,
tools = [ctx.attr._add_jar_manifest_entry.files_to_run],
)

compilejar = ctx.actions.declare_file("header_" + injar.basename, sibling = injar)
args = ctx.actions.args()
args.add_all(["--source", outjar, "--output", compilejar])
# We need to remove the `Class-Path` entry since bazel 4 forces `javac`
# to run `-Xlint:path` no matter what other flags are passed. Bazel
# manages the classpath for us, so the `Class-Path` manifest entry isn't
# needed. Worse, if it's there and the jars listed in it aren't found,
# the lint check will emit a `bad path element` warning. We get quiet and
# correct builds if we remove the `Class-Path` manifest entry entirely.
args.add_all(["--remove-entry", "Class-Path"])
ctx.actions.run(
executable = ctx.executable._add_jar_manifest_entry,
arguments = [args],
inputs = [outjar],
outputs = [compilejar],
mnemonic = "CreateCompileJar",
progress_message = "Creating compile jar for %s" % ctx.label,
)

if not ctx.attr._stamp_manifest[StampManifestProvider].stamp_enabled:
Expand All @@ -46,7 +54,7 @@ def _jvm_import_impl(ctx):
files = depset([outjar]),
),
JavaInfo(
compile_jar = outjar,
compile_jar = compilejar,
output_jar = outjar,
source_jar = ctx.file.srcjar,
deps = [
Expand Down Expand Up @@ -79,11 +87,11 @@ jvm_import = rule(
),
"_add_jar_manifest_entry": attr.label(
executable = True,
cfg = "host",
cfg = "exec",
default = "//private/tools/java/rules/jvm/external/jar:AddJarManifestEntry",
),
"_stamp_manifest": attr.label(
default = Label("@rules_jvm_external//settings:stamp_manifest"),
default = "@rules_jvm_external//settings:stamp_manifest",
),
},
implementation = _jvm_import_impl,
Expand Down
172 changes: 83 additions & 89 deletions private/tools/java/rules/jvm/external/jar/AddJarManifestEntry.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package rules.jvm.external.jar;

import rules.jvm.external.ByteStreams;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
Expand All @@ -8,18 +10,24 @@
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.List;
import java.util.Objects;
import java.util.jar.Attributes;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.jar.JarInputStream;
import java.util.jar.JarOutputStream;
import java.util.jar.Manifest;
import java.util.zip.CRC32;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;

import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
import static java.util.jar.Attributes.Name.MANIFEST_VERSION;

/*
* A class that will add an entry to the manifest and keep the same modification
# times of the jar entries.
Expand All @@ -43,7 +51,8 @@ public static void verboseLog(String logline) {
public static void main(String[] args) throws IOException {
Path out = null;
Path source = null;
String manifestEntry = null;
List<String> toAdd = new ArrayList<>();
List<String> toRemove = new ArrayList<>();

for (int i = 0; i < args.length; i++) {
switch (args[i]) {
Expand All @@ -52,7 +61,11 @@ public static void main(String[] args) throws IOException {
break;

case "--manifest-entry":
manifestEntry = args[++i];
toAdd.add(args[++i]);
break;

case "--remove-entry":
toRemove.add(args[++i]);
break;

case "--source":
Expand All @@ -67,37 +80,56 @@ public static void main(String[] args) throws IOException {
Objects.requireNonNull(source, "Source jar must be set.");
Objects.requireNonNull(out, "Output path must be set.");

addEntryToManifest(out, source, manifestEntry, false);
}
if (isJarSigned(source)) {
verboseLog("Signed jar. Will not modify: " + source);
Files.createDirectories(out.getParent());
Files.copy(source, out, REPLACE_EXISTING);
return;
}

public static void addEntryToManifest(Path out, Path source, String manifestEntry, boolean addManifest) throws IOException {
byte[] buffer = new byte[2048];
int bytesRead;
boolean manifestUpdated = false;
addEntryToManifest(out, source, toAdd, toRemove);
}

try (InputStream fis = Files.newInputStream(source);
ZipInputStream zis = new ZipInputStream(fis)) {
private static boolean isJarSigned(Path source) throws IOException {
try (InputStream is = Files.newInputStream(source);
JarInputStream jis = new JarInputStream(is)) {
for (ZipEntry entry = jis.getNextEntry(); entry != null; entry = jis.getNextJarEntry()) {
if (entry.isDirectory()) {
continue;
}
if (entry.getName().startsWith("META-INF/") && entry.getName().endsWith(".SF")) {
return true;
}
}
}
return false;
}

public static void addEntryToManifest(
Path out,
Path source,
List<String> toAdd,
List<String> toRemove) throws IOException {
try (JarFile jarFile = new JarFile(source.toFile(), false)) {
try (OutputStream fos = Files.newOutputStream(out);
ZipOutputStream zos = new JarOutputStream(fos)) {

if (addManifest) {
verboseLog("INFO: No jar manifest found in " + source + " Adding new jar manifest.");
Manifest manifest = new Manifest();
manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
manifest.getMainAttributes().put(new Attributes.Name("Created-By"), "AddJarManifestEntry");
String[] manifestEntryParts = manifestEntry.split(":", 2);
manifest.getMainAttributes().put(new Attributes.Name(manifestEntryParts[0]), manifestEntryParts[1]);

ZipEntry newManifestEntry = new ZipEntry(JarFile.MANIFEST_NAME);
newManifestEntry.setTime(DEFAULT_TIMESTAMP);
zos.putNextEntry(newManifestEntry);
manifest.write(zos);
manifestUpdated = true;
// Rewrite the manifest first
Manifest manifest = jarFile.getManifest();
if (manifest == null) {
manifest = new Manifest();
manifest.getMainAttributes().put(MANIFEST_VERSION, "1.0");
}
amendManifest(manifest, toAdd, toRemove);

ZipEntry newManifestEntry = new ZipEntry(JarFile.MANIFEST_NAME);
newManifestEntry.setTime(DEFAULT_TIMESTAMP);
zos.putNextEntry(newManifestEntry);
manifest.write(zos);

ZipEntry sourceEntry;
while ((sourceEntry = zis.getNextEntry()) != null) {
Enumeration<JarEntry> entries = jarFile.entries();
while (entries.hasMoreElements()) {
JarEntry sourceEntry = entries.nextElement();
String name = sourceEntry.getName();

ZipEntry outEntry = new ZipEntry(name);
Expand All @@ -106,76 +138,38 @@ public static void addEntryToManifest(Path out, Path source, String manifestEntr
outEntry.setComment(sourceEntry.getComment());
outEntry.setExtra(sourceEntry.getExtra());

if (manifestEntry != null && JarFile.MANIFEST_NAME.equals(name)) {
Manifest manifest = new Manifest(zis);
String[] manifestEntryParts = manifestEntry.split(":", 2);
manifest.getMainAttributes().put(new Attributes.Name(manifestEntryParts[0]), manifestEntryParts[1]);

if (sourceEntry.getMethod() == ZipEntry.STORED) {
CRC32OutputStream crc32OutputStream = new CRC32OutputStream();
manifest.write(crc32OutputStream);
outEntry.setSize(crc32OutputStream.getSize());
outEntry.setCrc(crc32OutputStream.getCRC());
}
zos.putNextEntry(outEntry);
manifest.write(zos);
manifestUpdated = true;

} else {
if (JarFile.MANIFEST_NAME.equals(name)) {
continue;
}

if (sourceEntry.getMethod() == ZipEntry.STORED) {
outEntry.setSize(sourceEntry.getSize());
outEntry.setCrc(sourceEntry.getCrc());
}
if (sourceEntry.getMethod() == ZipEntry.STORED) {
outEntry.setSize(sourceEntry.getSize());
outEntry.setCrc(sourceEntry.getCrc());
}

try {
zos.putNextEntry(outEntry);
} catch (ZipException e) {
if (e.getMessage().contains("duplicate entry:")) {
// If there is a duplicate entry we keep the first one we saw.
verboseLog("WARN: Skipping duplicate jar entry " + outEntry.getName() + " in " + source);
continue;
} else {
throw e;
}
}
while ((bytesRead = zis.read(buffer)) != -1) {
zos.write(buffer, 0, bytesRead);
try (InputStream in = jarFile.getInputStream(sourceEntry)) {
zos.putNextEntry(outEntry);
ByteStreams.copy(in, zos);
} catch (ZipException e) {
if (e.getMessage().contains("duplicate entry:")) {
// If there is a duplicate entry we keep the first one we saw.
verboseLog("WARN: Skipping duplicate jar entry " + outEntry.getName() + " in " + source);
continue;
} else {
throw e;
}
}
}
}
}

if (manifestEntry != null && !manifestUpdated) {
// If no manifest was found then re-run and add the MANIFEST.MF as the first entry in the output jar
addEntryToManifest(out, source, manifestEntry, true);
}
}

// OutputStream to find the CRC32 of an updated STORED zip entry
private static class CRC32OutputStream extends java.io.OutputStream {
private final CRC32 crc = new CRC32();
private long size = 0;

CRC32OutputStream() {}

public void write(int b) throws IOException {
crc.update(b);
size++;
}

public void write(byte[] b, int off, int len) throws IOException {
crc.update(b, off, len);
size += len;
}

public long getSize() {
return size;
}

public long getCRC() {
return crc.getValue();
}
private static void amendManifest(Manifest manifest, List<String> toAdd, List<String> toRemove) {
manifest.getMainAttributes().put(new Attributes.Name("Created-By"), "AddJarManifestEntry");
toAdd.forEach(manifestEntry -> {
String[] manifestEntryParts = manifestEntry.split(":", 2);
manifest.getMainAttributes().put(new Attributes.Name(manifestEntryParts[0]), manifestEntryParts[1]);
});
toRemove.forEach(name -> manifest.getMainAttributes().remove(new Attributes.Name(name)));
}
}
3 changes: 3 additions & 0 deletions private/tools/java/rules/jvm/external/jar/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ java_binary(
visibility = [
"//visibility:public",
],
deps = [
"//private/tools/java/rules/jvm/external:byte-streams",
],
)

java_binary(
Expand Down
Loading

0 comments on commit ec2c561

Please sign in to comment.