diff --git a/pom.xml b/pom.xml index dd1353ed75..e6316ecb6b 100644 --- a/pom.xml +++ b/pom.xml @@ -199,6 +199,7 @@ proto-google-cloud-storage-v2 gapic-google-cloud-storage-v2 google-cloud-storage-bom + storage-shared-benchmarking diff --git a/storage-shared-benchmarking/pom.xml b/storage-shared-benchmarking/pom.xml new file mode 100644 index 0000000000..3a4f575452 --- /dev/null +++ b/storage-shared-benchmarking/pom.xml @@ -0,0 +1,114 @@ + + + 4.0.0 + com.google.cloud + jar + storage-shared-benchmarking + 0.0.1-SNAPSHOT + + com.google.cloud + google-cloud-storage-parent + 2.26.2-SNAPSHOT + + + + 1.8 + 1.8 + UTF-8 + + + + info.picocli + picocli + 4.7.0 + + + com.google.cloud + google-cloud-storage + + + com.google.cloud + google-cloud-storage + 2.26.2-SNAPSHOT + tests + + + com.google.api + gax + + + com.google.api + api-common + + + com.google.guava + guava + + + com.google.cloud + google-cloud-core + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + ${uberjar.name} + + + com.google.cloud.storage.benchmarking.StorageSharedBenchmarkingCli + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + org.sonatype.plugins + nexus-staging-maven-plugin + + true + + + + + + \ No newline at end of file diff --git a/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/CloudMonitoringResult.java b/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/CloudMonitoringResult.java new file mode 100644 index 0000000000..f9884d12e5 --- /dev/null +++ b/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/CloudMonitoringResult.java @@ -0,0 +1,269 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.benchmarking; + +import com.google.common.base.MoreObjects; +import java.util.Objects; + +final class CloudMonitoringResult { + private final String library; + private final String api; + private final String op; + + private final int workers; + private final int object_size; + private final int app_buffer_size; + private final int chunksize; + private final boolean crc32c_enabled; + private final boolean md5_enabled; + private final int cpu_time_us; + private final String bucket_name; + private final String status; + private final String transfer_size; + private final String transfer_offset; + private final String failure_msg; + private final double throughput; + + CloudMonitoringResult( + String library, + String api, + String op, + int workers, + int objectSize, + int appBufferSize, + int chunksize, + boolean crc32cEnabled, + boolean md5Enabled, + int cpuTimeUs, + String bucketName, + String status, + String transferSize, + String transferOffset, + String failureMsg, + double throughput) { + this.library = library; + this.api = api; + this.op = op; + this.workers = workers; + this.object_size = objectSize; + this.app_buffer_size = appBufferSize; + this.chunksize = chunksize; + this.crc32c_enabled = crc32cEnabled; + this.md5_enabled = md5Enabled; + this.cpu_time_us = cpuTimeUs; + this.bucket_name = bucketName; + this.status = status; + this.transfer_size = transferSize; + this.transfer_offset = transferOffset; + this.failure_msg = failureMsg; + this.throughput = throughput; + } + + public static Builder newBuilder() { + return new Builder(); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("library", library) + .add("api", api) + .add("op", op) + .add("workers", workers) + .add("object_size", object_size) + .add("app_buffer_size", app_buffer_size) + .add("chunksize", chunksize) + .add("crc32c_enabled", crc32c_enabled) + .add("md5_enabled", md5_enabled) + .add("cpu_time_us", cpu_time_us) + .add("bucket_name", bucket_name) + .add("status", status) + .add("transfer_size", transfer_size) + .add("transfer_offset", transfer_offset) + .add("failure_msg", failure_msg) + .add("throughput", throughput) + .toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CloudMonitoringResult)) { + return false; + } + CloudMonitoringResult result = (CloudMonitoringResult) o; + return workers == result.workers + && object_size == result.object_size + && app_buffer_size == result.app_buffer_size + && chunksize == result.chunksize + && crc32c_enabled == result.crc32c_enabled + && md5_enabled == result.md5_enabled + && cpu_time_us == result.cpu_time_us + && Double.compare(result.throughput, throughput) == 0 + && Objects.equals(library, result.library) + && Objects.equals(api, result.api) + && Objects.equals(op, result.op) + && Objects.equals(bucket_name, result.bucket_name) + && Objects.equals(status, result.status) + && Objects.equals(transfer_size, result.transfer_size) + && Objects.equals(transfer_offset, result.transfer_offset) + && Objects.equals(failure_msg, result.failure_msg); + } + + @Override + public int hashCode() { + return Objects.hash( + library, + api, + op, + workers, + object_size, + app_buffer_size, + chunksize, + crc32c_enabled, + md5_enabled, + cpu_time_us, + bucket_name, + status, + transfer_size, + transfer_offset, + failure_msg, + throughput); + } + + public static class Builder { + + private String library; + private String api; + private String op; + private int workers; + private int objectSize; + private int appBufferSize; + private int chunksize; + private boolean crc32cEnabled; + private boolean md5Enabled; + private int cpuTimeUs; + private String bucketName; + private String status; + private String transferSize; + private String transferOffset; + private String failureMsg; + private double throughput; + + public Builder setLibrary(String library) { + this.library = library; + return this; + } + + public Builder setApi(String api) { + this.api = api; + return this; + } + + public Builder setOp(String op) { + this.op = op; + return this; + } + + public Builder setWorkers(int workers) { + this.workers = workers; + return this; + } + + public Builder setObjectSize(int objectSize) { + this.objectSize = objectSize; + return this; + } + + public Builder setAppBufferSize(int appBufferSize) { + this.appBufferSize = appBufferSize; + return this; + } + + public Builder setChunksize(int chunksize) { + this.chunksize = chunksize; + return this; + } + + public Builder setCrc32cEnabled(boolean crc32cEnabled) { + this.crc32cEnabled = crc32cEnabled; + return this; + } + + public Builder setMd5Enabled(boolean md5Enabled) { + this.md5Enabled = md5Enabled; + return this; + } + + public Builder setCpuTimeUs(int cpuTimeUs) { + this.cpuTimeUs = cpuTimeUs; + return this; + } + + public Builder setBucketName(String bucketName) { + this.bucketName = bucketName; + return this; + } + + public Builder setStatus(String status) { + this.status = status; + return this; + } + + public Builder setTransferSize(String transferSize) { + this.transferSize = transferSize; + return this; + } + + public Builder setTransferOffset(String transferOffset) { + this.transferOffset = transferOffset; + return this; + } + + public Builder setFailureMsg(String failureMsg) { + this.failureMsg = failureMsg; + return this; + } + + public Builder setThroughput(double throughput) { + this.throughput = throughput; + return this; + } + + public CloudMonitoringResult build() { + return new CloudMonitoringResult( + library, + api, + op, + workers, + objectSize, + appBufferSize, + chunksize, + crc32cEnabled, + md5Enabled, + cpuTimeUs, + bucketName, + status, + transferSize, + transferOffset, + failureMsg, + throughput); + } + } +} diff --git a/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingCli.java b/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingCli.java new file mode 100644 index 0000000000..387d39348f --- /dev/null +++ b/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingCli.java @@ -0,0 +1,161 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.benchmarking; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.api.core.ListenableFutureToApiFuture; +import com.google.api.gax.retrying.RetrySettings; +import com.google.api.gax.rpc.ApiExceptions; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.DataGenerator; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import com.google.cloud.storage.TmpFile; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.Executors; +import java.util.regex.Pattern; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +@Command(name = "ssb") +public final class StorageSharedBenchmarkingCli implements Runnable { + // TODO: check what input validation is needed for option values. + @Option(names = "-project", description = "GCP Project Identifier", required = true) + String project; + + @Option(names = "-bucket", description = "Name of the bucket to use", required = true) + String bucket; + + @Option(names = "-samples", defaultValue = "8000", description = "Number of samples to report") + int samples; + + @Option( + names = "-workers", + defaultValue = "16", + description = "Number of workers to run in parallel for the workload") + int workers; + + @Option(names = "-api", defaultValue = "JSON", description = "API to use") + String api; + + @Option( + names = "-object_size", + defaultValue = "1048576..1048576", + description = + "any positive integer, or an inclusive range such as min..max where min and max are positive integers") + String objectSize; + + @Option( + names = "-output_type", + defaultValue = "cloud-monitoring", + description = "Output results format") + String outputType; + + @Option( + names = "-test_type", + description = "Specify which workload the cli should run", + required = true) + String testType; + + public static void main(String[] args) { + CommandLine cmd = new CommandLine(StorageSharedBenchmarkingCli.class); + System.exit(cmd.execute(args)); + } + + @Override + public void run() { + switch (testType) { + case "w1r3": + runWorkload1(); + break; + default: + throw new IllegalStateException("Specify a workload to run"); + } + } + + private void runWorkload1() { + RetrySettings retrySettings = StorageOptions.getDefaultRetrySettings().toBuilder().build(); + + StorageOptions retryStorageOptions = + StorageOptions.newBuilder().setProjectId(project).setRetrySettings(retrySettings).build(); + Storage storageClient = retryStorageOptions.getService(); + Path tempDir = Paths.get(System.getProperty("java.io.tmpdir")); + ListeningExecutorService executorService = + MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(workers)); + List> workloadRuns = new ArrayList<>(); + Range objectSizeRange = Range.of(objectSize); + for (int i = 0; i < samples; i++) { + try { + TmpFile file = + DataGenerator.base64Characters() + .tempFile(tempDir, getRandomInt(objectSizeRange.min, objectSizeRange.max)); + BlobInfo blob = BlobInfo.newBuilder(bucket, file.toString()).build(); + workloadRuns.add( + convert( + executorService.submit(new Workload1(file, blob, storageClient, workers, api)))); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + ApiExceptions.callAndTranslateApiException(ApiFutures.allAsList(workloadRuns)); + } + + public static int getRandomInt(int min, int max) { + if (min == max) return min; + Random random = new Random(); + return random.nextInt((max - min) + 1) + min; + } + + private static ApiFuture convert(ListenableFuture lf) { + return new ListenableFutureToApiFuture<>(lf); + } + + private static final class Range { + private final int min; + private final int max; + + private Range(int min, int max) { + this.min = min; + this.max = max; + } + + public static Range of(int min, int max) { + return new Range(min, max); + } + // Takes an object size range of format min..max and creates a range object + public static Range of(String range) { + Pattern p = Pattern.compile("\\.\\."); + String[] splitRangeVals = p.split(range); + if (splitRangeVals.length == 2) { + String min = splitRangeVals[0]; + String max = splitRangeVals[1]; + return of(Integer.parseInt(min), Integer.parseInt(max)); + } + throw new IllegalStateException("Expected a size range of format min..max, but got " + range); + } + } +} diff --git a/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingUtils.java b/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingUtils.java new file mode 100644 index 0000000000..0407cf7015 --- /dev/null +++ b/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingUtils.java @@ -0,0 +1,36 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage.benchmarking; + +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Storage; +import java.time.Duration; + +class StorageSharedBenchmarkingUtils { + public static long SSB_SIZE_THRESHOLD_BYTES = 1048576; + public static int DEFAULT_NUMBER_OF_READS = 3; + + public static void cleanupObject(Storage storage, Blob created) { + storage.delete( + created.getBlobId(), Storage.BlobSourceOption.generationMatch(created.getGeneration())); + } + + public static double calculateThroughput(long size, Duration elapsedTime) { + return size >= StorageSharedBenchmarkingUtils.SSB_SIZE_THRESHOLD_BYTES + ? size / 1024 / 1024 / (elapsedTime.toNanos()) + : size / 1024 / (elapsedTime.toNanos()); + } +} diff --git a/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/Workload1.java b/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/Workload1.java new file mode 100644 index 0000000000..9d3de47292 --- /dev/null +++ b/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/Workload1.java @@ -0,0 +1,100 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.benchmarking; + +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.TmpFile; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.Callable; + +final class Workload1 implements Callable { + private final TmpFile file; + private final BlobInfo blob; + private final Storage storage; + private final int workers; + private final String api; + + Workload1(TmpFile file, BlobInfo blob, Storage storage, int workers, String api) { + this.file = file; + this.blob = blob; + this.storage = storage; + this.workers = workers; + this.api = api; + } + + @Override + public String call() throws Exception { + Clock clock = Clock.systemDefaultZone(); + + // Get the start time + Instant startTime = clock.instant(); + Blob created = storage.createFrom(blob, file.getPath()); + Instant endTime = clock.instant(); + Duration elapsedTimeUpload = Duration.between(startTime, endTime); + System.out.println( + generateCloudMonitoringResult( + "WRITE", + StorageSharedBenchmarkingUtils.calculateThroughput( + created.getSize().longValue(), elapsedTimeUpload), + created) + .toString()); + Path tempDir = Paths.get(System.getProperty("java.io.tmpdir")); + for (int i = 0; i <= StorageSharedBenchmarkingUtils.DEFAULT_NUMBER_OF_READS; i++) { + TmpFile dest = TmpFile.of(tempDir, "prefix", "bin"); + startTime = clock.instant(); + storage.downloadTo(created.getBlobId(), dest.getPath()); + endTime = clock.instant(); + Duration elapsedTimeDownload = Duration.between(startTime, endTime); + System.out.println( + generateCloudMonitoringResult( + "READ[" + i + "]", + StorageSharedBenchmarkingUtils.calculateThroughput( + created.getSize().longValue(), elapsedTimeDownload), + created) + .toString()); + } + StorageSharedBenchmarkingUtils.cleanupObject(storage, created); + return "OK"; + } + + private CloudMonitoringResult generateCloudMonitoringResult( + String op, double throughput, Blob created) { + CloudMonitoringResult result = + CloudMonitoringResult.newBuilder() + .setLibrary("java") + .setApi(api) + .setOp(op) + .setWorkers(workers) + .setObjectSize(created.getSize().intValue()) + .setChunksize(created.getSize().intValue()) + .setCrc32cEnabled(false) + .setMd5Enabled(false) + .setCpuTimeUs(-1) + .setBucketName(created.getBucket()) + .setStatus("OK") + .setTransferSize(created.getSize().toString()) + .setThroughput(throughput) + .build(); + return result; + } +}