From bf2cdf0ef4e9be25b4337a58479184dd1a577620 Mon Sep 17 00:00:00 2001 From: Son Luong Ngoc Date: Sat, 29 Apr 2023 13:32:05 +0200 Subject: [PATCH] Conditionally set output_paths based on Remote Executor capabilities This is a follow up to #18269, toward the discussion in #18202. Bump the Remote API supported version to v2.1. Based on the Capability of the Remote Executor, either use output_paths field or the legacy fields output_files and output_directories. --- .../devtools/build/lib/remote/ApiVersion.java | 7 +- .../lib/remote/RemoteExecutionService.java | 51 +++++--- .../remote/RemoteExecutionServiceTest.java | 122 +++++++++++++++++- ...SpawnRunnerWithGrpcRemoteExecutorTest.java | 3 +- .../build/remote/worker/ExecutionServer.java | 27 ++-- 5 files changed, 169 insertions(+), 41 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/remote/ApiVersion.java b/src/main/java/com/google/devtools/build/lib/remote/ApiVersion.java index ee4ed593a81827..d1cb71f91b7456 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/ApiVersion.java +++ b/src/main/java/com/google/devtools/build/lib/remote/ApiVersion.java @@ -28,7 +28,12 @@ public class ApiVersion implements Comparable { public static final ApiVersion low = new ApiVersion(SemVer.newBuilder().setMajor(2).setMinor(0).build()); public static final ApiVersion high = - new ApiVersion(SemVer.newBuilder().setMajor(2).setMinor(0).build()); + new ApiVersion(SemVer.newBuilder().setMajor(2).setMinor(1).build()); + + // The version of the Remote Execution API that starts supporting the + // Command.output_paths and ActionResult.output_symlinks fields. + public static final ApiVersion twoPointOne = + new ApiVersion(SemVer.newBuilder().setMajor(2).setMinor(1).build()); public ApiVersion(int major, int minor, int patch, String prerelease) { this.major = major; diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java index fd11cf0c2c13ea..c046f054515be8 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java @@ -234,6 +234,7 @@ public RemoteExecutionService( } private Command buildCommand( + boolean useOutputPaths, Collection outputs, List arguments, ImmutableMap env, @@ -241,24 +242,30 @@ private Command buildCommand( RemotePathResolver remotePathResolver, @Nullable SpawnScrubber spawnScrubber) { Command.Builder command = Command.newBuilder(); - ArrayList outputFiles = new ArrayList<>(); - ArrayList outputDirectories = new ArrayList<>(); - ArrayList outputPaths = new ArrayList<>(); - for (ActionInput output : outputs) { - String pathString = decodeBytestringUtf8(remotePathResolver.localPathToOutputPath(output)); - if (output.isDirectory()) { - outputDirectories.add(pathString); - } else { - outputFiles.add(pathString); + if (useOutputPaths) { + var outputPaths = new ArrayList(); + for (ActionInput output : outputs) { + String pathString = decodeBytestringUtf8(remotePathResolver.localPathToOutputPath(output)); + outputPaths.add(pathString); + } + Collections.sort(outputPaths); + command.addAllOutputPaths(outputPaths); + } else { + var outputFiles = new ArrayList(); + var outputDirectories = new ArrayList(); + for (ActionInput output : outputs) { + String pathString = decodeBytestringUtf8(remotePathResolver.localPathToOutputPath(output)); + if (output.isDirectory()) { + outputDirectories.add(pathString); + } else { + outputFiles.add(pathString); + } } - outputPaths.add(pathString); + Collections.sort(outputFiles); + Collections.sort(outputDirectories); + command.addAllOutputFiles(outputFiles); + command.addAllOutputDirectories(outputDirectories); } - Collections.sort(outputFiles); - Collections.sort(outputDirectories); - Collections.sort(outputPaths); - command.addAllOutputFiles(outputFiles); - command.addAllOutputDirectories(outputDirectories); - command.addAllOutputPaths(outputPaths); if (platform != null) { command.setPlatform(platform); @@ -585,8 +592,20 @@ public RemoteAction buildRemoteAction(Spawn spawn, SpawnExecutionContext context platform = PlatformUtils.getPlatformProto(spawn, remoteOptions); } + var useOutputPaths = true; + if (mayBeExecutedRemotely(spawn)) { + var capabilities = remoteExecutor.getServerCapabilities(); + if (capabilities != null) { + var supportStatus = ClientApiVersion.current.checkServerSupportedVersions(capabilities); + if (supportStatus.isSupported()) { + useOutputPaths = + supportStatus.getHighestSupportedVersion().compareTo(ApiVersion.twoPointOne) >= 0; + } + } + } Command command = buildCommand( + useOutputPaths, spawn.getOutputFiles(), spawn.getArguments(), spawn.getEnvironment(), diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java index 95124d005b61cd..2d9a569978c4a5 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java @@ -38,6 +38,7 @@ import build.bazel.remote.execution.v2.Digest; import build.bazel.remote.execution.v2.Directory; import build.bazel.remote.execution.v2.DirectoryNode; +import build.bazel.remote.execution.v2.ExecutionCapabilities; import build.bazel.remote.execution.v2.FileNode; import build.bazel.remote.execution.v2.NodeProperties; import build.bazel.remote.execution.v2.NodeProperty; @@ -46,9 +47,11 @@ import build.bazel.remote.execution.v2.OutputSymlink; import build.bazel.remote.execution.v2.Platform; import build.bazel.remote.execution.v2.RequestMetadata; +import build.bazel.remote.execution.v2.ServerCapabilities; import build.bazel.remote.execution.v2.SymlinkAbsolutePathStrategy; import build.bazel.remote.execution.v2.SymlinkNode; import build.bazel.remote.execution.v2.Tree; +import build.bazel.semver.SemVer; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableClassToInstanceMap; import com.google.common.collect.ImmutableList; @@ -153,6 +156,23 @@ public class RemoteExecutionServiceTest { private final Reporter reporter = new Reporter(new EventBus()); private final StoredEventHandler eventHandler = new StoredEventHandler(); + // In the past, Bazel only supports RemoteApi version 2.0. + // Use this to ensure we are backward compatible with Servers that only support 2.0. + private final ApiVersion oldApiVersion = new ApiVersion(SemVer.newBuilder().setMajor(2).build()); + private final ServerCapabilities legacyRemoteExecutorCapabilities = + ServerCapabilities.newBuilder() + .setLowApiVersion(oldApiVersion.toSemVer()) + .setHighApiVersion(oldApiVersion.toSemVer()) + .setExecutionCapabilities(ExecutionCapabilities.newBuilder().setExecEnabled(true).build()) + .build(); + + private final ServerCapabilities remoteExecutorCapabilities = + ServerCapabilities.newBuilder() + .setLowApiVersion(ApiVersion.low.toSemVer()) + .setHighApiVersion(ApiVersion.high.toSemVer()) + .setExecutionCapabilities(ExecutionCapabilities.newBuilder().setExecEnabled(true).build()) + .build(); + RemoteOptions remoteOptions; private FileSystem fs; private Path execRoot; @@ -197,6 +217,7 @@ public final void setUp() throws Exception { cache = spy(new InMemoryRemoteCache(spy(new InMemoryCacheClient()), remoteOptions, digestUtil)); executor = mock(RemoteExecutionClient.class); + when(executor.getServerCapabilities()).thenReturn(remoteExecutorCapabilities); RequestMetadata metadata = TracingMetadataUtils.buildMetadata("none", "none", "action-id", null); @@ -215,9 +236,27 @@ public void buildRemoteAction_withRegularFileAsOutput() throws Exception { RemoteAction remoteAction = service.buildRemoteAction(spawn, context); - assertThat(remoteAction.getCommand().getOutputFilesList()).containsExactly(execPath.toString()); + assertThat(remoteAction.getCommand().getOutputFilesList()).isEmpty(); + assertThat(remoteAction.getCommand().getOutputDirectoriesList()).isEmpty(); assertThat(remoteAction.getCommand().getOutputPathsList()).containsExactly(execPath.toString()); + } + + @Test + public void legacy_buildRemoteAction_withRegularFileAsOutput() throws Exception { + when(executor.getServerCapabilities()).thenReturn(legacyRemoteExecutorCapabilities); + PathFragment execPath = execRoot.getRelative("path/to/tree").asFragment(); + Spawn spawn = + new SpawnBuilder("dummy") + .withOutput(ActionsTestUtil.createArtifactWithExecPath(artifactRoot, execPath)) + .build(); + FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn); + RemoteExecutionService service = newRemoteExecutionService(); + + RemoteAction remoteAction = service.buildRemoteAction(spawn, context); + + assertThat(remoteAction.getCommand().getOutputFilesList()).containsExactly(execPath.toString()); assertThat(remoteAction.getCommand().getOutputDirectoriesList()).isEmpty(); + assertThat(remoteAction.getCommand().getOutputPathsList()).isEmpty(); } @Test @@ -233,8 +272,28 @@ public void buildRemoteAction_withTreeArtifactAsOutput() throws Exception { RemoteAction remoteAction = service.buildRemoteAction(spawn, context); + assertThat(remoteAction.getCommand().getOutputFilesList()).isEmpty(); + assertThat(remoteAction.getCommand().getOutputDirectoriesList()).isEmpty(); + assertThat(remoteAction.getCommand().getOutputPathsList()).containsExactly("path/to/dir"); + } + + @Test + public void legacy_buildRemoteAction_withTreeArtifactAsOutput() throws Exception { + when(executor.getServerCapabilities()).thenReturn(legacyRemoteExecutorCapabilities); + Spawn spawn = + new SpawnBuilder("dummy") + .withOutput( + ActionsTestUtil.createTreeArtifactWithGeneratingAction( + artifactRoot, PathFragment.create("path/to/dir"))) + .build(); + FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn); + RemoteExecutionService service = newRemoteExecutionService(); + + RemoteAction remoteAction = service.buildRemoteAction(spawn, context); + assertThat(remoteAction.getCommand().getOutputFilesList()).isEmpty(); assertThat(remoteAction.getCommand().getOutputDirectoriesList()).containsExactly("path/to/dir"); + assertThat(remoteAction.getCommand().getOutputPathsList()).isEmpty(); } @Test @@ -250,11 +309,30 @@ public void buildRemoteAction_withUnresolvedSymlinkAsOutput() throws Exception { RemoteAction remoteAction = service.buildRemoteAction(spawn, context); - assertThat(remoteAction.getCommand().getOutputFilesList()).containsExactly("path/to/link"); + assertThat(remoteAction.getCommand().getOutputFilesList()).isEmpty(); assertThat(remoteAction.getCommand().getOutputDirectoriesList()).isEmpty(); assertThat(remoteAction.getCommand().getOutputPathsList()).containsExactly("path/to/link"); } + @Test + public void legacy_buildRemoteAction_withUnresolvedSymlinkAsOutput() throws Exception { + when(executor.getServerCapabilities()).thenReturn(legacyRemoteExecutorCapabilities); + Spawn spawn = + new SpawnBuilder("dummy") + .withOutput( + ActionsTestUtil.createUnresolvedSymlinkArtifactWithExecPath( + artifactRoot, PathFragment.create("path/to/link"))) + .build(); + FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn); + RemoteExecutionService service = newRemoteExecutionService(); + + RemoteAction remoteAction = service.buildRemoteAction(spawn, context); + + assertThat(remoteAction.getCommand().getOutputFilesList()).containsExactly("path/to/link"); + assertThat(remoteAction.getCommand().getOutputDirectoriesList()).isEmpty(); + assertThat(remoteAction.getCommand().getOutputPathsList()).isEmpty(); + } + @Test public void buildRemoteAction_withActionInputAsOutput() throws Exception { Spawn spawn = @@ -266,8 +344,42 @@ public void buildRemoteAction_withActionInputAsOutput() throws Exception { RemoteAction remoteAction = service.buildRemoteAction(spawn, context); + assertThat(remoteAction.getCommand().getOutputFilesList()).isEmpty(); + assertThat(remoteAction.getCommand().getOutputDirectoriesList()).isEmpty(); + assertThat(remoteAction.getCommand().getOutputPathsList()).containsExactly("path/to/file"); + } + + @Test + public void legacy_buildRemoteAction_withActionInputFileAsOutput() throws Exception { + when(executor.getServerCapabilities()).thenReturn(legacyRemoteExecutorCapabilities); + Spawn spawn = + new SpawnBuilder("dummy") + .withOutput(ActionInputHelper.fromPath(PathFragment.create("path/to/file"))) + .build(); + FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn); + RemoteExecutionService service = newRemoteExecutionService(); + + RemoteAction remoteAction = service.buildRemoteAction(spawn, context); + assertThat(remoteAction.getCommand().getOutputFilesList()).containsExactly("path/to/file"); assertThat(remoteAction.getCommand().getOutputDirectoriesList()).isEmpty(); + assertThat(remoteAction.getCommand().getOutputPathsList()).isEmpty(); + } + + @Test + public void buildRemoteAction_withActionInputDirectoryAsOutput() throws Exception { + Spawn spawn = + new SpawnBuilder("dummy") + .withOutput(ActionInputHelper.fromPath(PathFragment.create("path/to/dir"))) + .build(); + FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn); + RemoteExecutionService service = newRemoteExecutionService(); + + RemoteAction remoteAction = service.buildRemoteAction(spawn, context); + + assertThat(remoteAction.getCommand().getOutputFilesList()).isEmpty(); + assertThat(remoteAction.getCommand().getOutputDirectoriesList()).isEmpty(); + assertThat(remoteAction.getCommand().getOutputPathsList()).containsExactly("path/to/dir"); } @Test @@ -2373,10 +2485,8 @@ public void buildRemoteActionWithPathMapping(@TestParameter boolean remoteMerkle .containsExactly( PathFragment.create("outputs/bin/input1"), mappedInput, PathFragment.create("outputs/bin/input2"), unmappedInput); - assertThat(remoteAction.getCommand().getOutputFilesList()) - .containsExactly("outputs/bin/dir/output1", "outputs/bin/other_dir/output2"); - assertThat(remoteAction.getCommand().getOutputDirectoriesList()) - .containsExactly("outputs/bin/output_dir"); + assertThat(remoteAction.getCommand().getOutputFilesList()).isEmpty(); + assertThat(remoteAction.getCommand().getOutputDirectoriesList()).isEmpty(); assertThat(remoteAction.getCommand().getOutputPathsList()) .containsExactly( "outputs/bin/dir/output1", "outputs/bin/other_dir/output2", "outputs/bin/output_dir"); diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java index 1b02b2e5240781..fa99e7a84847b0 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java @@ -311,6 +311,8 @@ public Single create() { .build(); ServerCapabilities caps = ServerCapabilities.newBuilder() + .setLowApiVersion(ApiVersion.low.toSemVer()) + .setHighApiVersion(ApiVersion.high.toSemVer()) .setExecutionCapabilities( ExecutionCapabilities.newBuilder().setExecEnabled(true).build()) .build(); @@ -371,7 +373,6 @@ public int maxConcurrency() { .setName("VARIABLE") .setValue("value") .build()) - .addAllOutputFiles(ImmutableList.of("bar", "foo")) .addAllOutputPaths(ImmutableList.of("bar", "foo")) .build(); cmdDigest = DIGEST_UTIL.compute(command); diff --git a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/ExecutionServer.java b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/ExecutionServer.java index cac99e3adc05f6..ad3399f60d403d 100644 --- a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/ExecutionServer.java +++ b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/ExecutionServer.java @@ -284,28 +284,21 @@ private ActionResult execute( Path workingDirectory = execRoot.getRelative(command.getWorkingDirectory()); workingDirectory.createDirectoryAndParents(); - List outputs = - new ArrayList<>(command.getOutputDirectoriesCount() + command.getOutputFilesCount()); + List outputs = new ArrayList<>(command.getOutputPathsCount()); - for (String output : command.getOutputFilesList()) { + for (String output : command.getOutputPathsList()) { Path file = workingDirectory.getRelative(output); - if (file.exists()) { + // Since https://github.com/bazelbuild/bazel/pull/15818, + // Bazel includes all expected output directories as part of Action's inputs. + // + // Ensure no output file exists before execution happen. + // Ignore if output directories pre-exist. + if (file.exists() && !file.isDirectory()) { throw new FileAlreadyExistsException("Output file already exists: " + file); } file.getParentDirectory().createDirectoryAndParents(); outputs.add(file); } - for (String output : command.getOutputDirectoriesList()) { - Path file = workingDirectory.getRelative(output); - if (file.exists()) { - if (!file.isDirectory()) { - throw new FileAlreadyExistsException( - "Non-directory exists at output directory path: " + file); - } - } - file.getParentDirectory().createDirectoryAndParents(); - outputs.add(file); - } // TODO(ulfjack): This is basically a copy of LocalSpawnRunner. Ideally, we'd use that // implementation instead of copying it. @@ -432,8 +425,8 @@ private static long getUid() throws InterruptedException { com.google.devtools.build.lib.shell.Command cmd = new com.google.devtools.build.lib.shell.Command( new String[] {"id", "-u"}, - /*environmentVariables=*/ null, - /*workingDirectory=*/ null, + /* environmentVariables= */ null, + /* workingDirectory= */ null, uidTimeout); try { ByteArrayOutputStream stdout = new ByteArrayOutputStream();