From 77df0767c67a0209e50cf3ffe9759954b609341a Mon Sep 17 00:00:00 2001 From: Tamas Cservenak Date: Mon, 5 Aug 2024 14:35:50 +0200 Subject: [PATCH] MinIO --- maven-resolver-named-locks-redisson/pom.xml | 4 - maven-resolver-transport-minio/pom.xml | 151 +++++++++++ .../transport/minio/MinioTransporter.java | 203 +++++++++++++++ .../MinioTransporterConfigurationKeys.java | 59 +++++ .../minio/MinioTransporterFactory.java | 99 ++++++++ .../aether/transport/minio/ObjectName.java | 77 ++++++ .../minio/ObjectNameMapperFactory.java | 28 +++ .../FixedBucketObjectNameMapperFactory.java | 51 ++++ .../RepositoryIdObjectNameMapperFactory.java | 44 ++++ .../aether/transport/minio/package-info.java | 29 +++ .../minio/MinioTransporterFactoryTest.java | 104 ++++++++ .../transport/minio/MinioTransporterIT.java | 237 ++++++++++++++++++ pom.xml | 2 + src/site/markdown/configuration.md | 24 +- 14 files changed, 1097 insertions(+), 15 deletions(-) create mode 100644 maven-resolver-transport-minio/pom.xml create mode 100644 maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/MinioTransporter.java create mode 100644 maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/MinioTransporterConfigurationKeys.java create mode 100644 maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/MinioTransporterFactory.java create mode 100644 maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/ObjectName.java create mode 100644 maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/ObjectNameMapperFactory.java create mode 100644 maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/internal/FixedBucketObjectNameMapperFactory.java create mode 100644 maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/internal/RepositoryIdObjectNameMapperFactory.java create mode 100644 maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/package-info.java create mode 100644 maven-resolver-transport-minio/src/test/java/org/eclipse/aether/transport/minio/MinioTransporterFactoryTest.java create mode 100644 maven-resolver-transport-minio/src/test/java/org/eclipse/aether/transport/minio/MinioTransporterIT.java diff --git a/maven-resolver-named-locks-redisson/pom.xml b/maven-resolver-named-locks-redisson/pom.xml index d8546fc73..336d8135c 100644 --- a/maven-resolver-named-locks-redisson/pom.xml +++ b/maven-resolver-named-locks-redisson/pom.xml @@ -31,10 +31,6 @@ Maven Artifact Resolver Named Locks using Redisson A synchronization utility implementation using Redisson. - - 1.20.1 - - org.apache.maven.resolver diff --git a/maven-resolver-transport-minio/pom.xml b/maven-resolver-transport-minio/pom.xml new file mode 100644 index 000000000..3dc6c6a29 --- /dev/null +++ b/maven-resolver-transport-minio/pom.xml @@ -0,0 +1,151 @@ + + + + 4.0.0 + + + org.apache.maven.resolver + maven-resolver + 2.0.2-SNAPSHOT + + + maven-resolver-transport-minio + jar + + Maven Artifact Resolver Transport S3 MinIO + Maven Artifact Transport S3 MinIO (Java). + + + 8 + + + + + org.slf4j + slf4j-api + + + org.apache.maven.resolver + maven-resolver-api + + + org.apache.maven.resolver + maven-resolver-spi + + + org.apache.maven.resolver + maven-resolver-util + + + javax.inject + javax.inject + provided + true + + + + io.minio + minio + 8.5.11 + + + + + commons-codec + commons-codec + 1.17.1 + runtime + + + org.apache.commons + commons-compress + 1.26.2 + runtime + + + org.bouncycastle + bcprov-jdk18on + 1.78.1 + runtime + + + + org.junit.jupiter + junit-jupiter-api + test + + + org.slf4j + slf4j-simple + test + + + org.apache.maven.resolver + maven-resolver-test-util + test + + + org.apache.maven.resolver + maven-resolver-impl + test + + + org.testcontainers + testcontainers + ${testcontainersVersion} + test + + + org.testcontainers + junit-jupiter + ${testcontainersVersion} + test + + + org.testcontainers + minio + ${testcontainersVersion} + test + + + + + + + org.eclipse.sisu + sisu-maven-plugin + + + + + + + run-its + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + + + + diff --git a/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/MinioTransporter.java b/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/MinioTransporter.java new file mode 100644 index 000000000..fcb920689 --- /dev/null +++ b/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/MinioTransporter.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.minio; + +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +import io.minio.GetObjectArgs; +import io.minio.MinioClient; +import io.minio.StatObjectArgs; +import io.minio.UploadObjectArgs; +import io.minio.errors.ErrorResponseException; +import org.eclipse.aether.ConfigurationProperties; +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.repository.AuthenticationContext; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.spi.connector.transport.AbstractTransporter; +import org.eclipse.aether.spi.connector.transport.GetTask; +import org.eclipse.aether.spi.connector.transport.PeekTask; +import org.eclipse.aether.spi.connector.transport.PutTask; +import org.eclipse.aether.spi.connector.transport.Transporter; +import org.eclipse.aether.transfer.NoTransporterException; +import org.eclipse.aether.util.ConfigUtils; +import org.eclipse.aether.util.FileUtils; + +/** + * A transporter for S3 backed by MinIO Java. + * + * @since 2.0.2 + */ +final class MinioTransporter extends AbstractTransporter implements Transporter { + private final URI baseUri; + + private final MinioClient client; + + private final Function objectNameMapper; + + private final Map headers; + + MinioTransporter( + RepositorySystemSession session, RemoteRepository repository, Function objectNameMapper) + throws NoTransporterException { + this.objectNameMapper = objectNameMapper; + try { + URI uri = new URI(repository.getUrl()).parseServerAuthority(); + if (uri.isOpaque()) { + throw new URISyntaxException(repository.getUrl(), "URL must not be opaque"); + } + if (uri.getRawFragment() != null || uri.getRawQuery() != null) { + throw new URISyntaxException(repository.getUrl(), "URL must not have fragment or query"); + } + String path = uri.getPath(); + if (path == null) { + path = "/"; + } + if (!path.startsWith("/")) { + path = "/" + path; + } + if (!path.endsWith("/")) { + path = path + "/"; + } + this.baseUri = URI.create(uri.getScheme() + "://" + uri.getRawAuthority() + path); + } catch (URISyntaxException e) { + throw new NoTransporterException(repository, e.getMessage(), e); + } + + HashMap headers = new HashMap<>(); + @SuppressWarnings("unchecked") + Map configuredHeaders = (Map) ConfigUtils.getMap( + session, + Collections.emptyMap(), + ConfigurationProperties.HTTP_HEADERS + "." + repository.getId(), + ConfigurationProperties.HTTP_HEADERS); + if (configuredHeaders != null) { + configuredHeaders.forEach((k, v) -> headers.put(String.valueOf(k), v != null ? String.valueOf(v) : null)); + } + this.headers = headers; + + String username = null; + String password = null; + try (AuthenticationContext repoAuthContext = AuthenticationContext.forRepository(session, repository)) { + if (repoAuthContext != null) { + username = repoAuthContext.get(AuthenticationContext.USERNAME); + password = repoAuthContext.get(AuthenticationContext.PASSWORD); + } + } + if (username == null || password == null) { + throw new NoTransporterException(repository, "No accessKey and/or secretKey provided"); + } + + try { + this.client = MinioClient.builder() + .endpoint(repository.getUrl()) + .credentials(username, password) + .build(); + } catch (Exception e) { + throw new NoTransporterException(repository, e); + } + } + + @Override + public int classify(Throwable error) { + if (error instanceof ErrorResponseException) { + String errorCode = ((ErrorResponseException) error).errorResponse().code(); + if ("NoSuchKey".equals(errorCode) || "NoSuchBucket".equals(errorCode)) { + return ERROR_NOT_FOUND; + } + } + return ERROR_OTHER; + } + + @Override + protected void implPeek(PeekTask task) throws Exception { + ObjectName objectName = + objectNameMapper.apply(baseUri.relativize(task.getLocation()).getPath()); + StatObjectArgs.Builder builder = StatObjectArgs.builder() + .bucket(objectName.getBucket()) + .object(objectName.getName()) + .extraHeaders(headers); + client.statObject(builder.build()); + } + + @Override + protected void implGet(GetTask task) throws Exception { + ObjectName objectName = + objectNameMapper.apply(baseUri.relativize(task.getLocation()).getPath()); + try (InputStream stream = client.getObject(GetObjectArgs.builder() + .bucket(objectName.getBucket()) + .object(objectName.getName()) + .extraHeaders(headers) + .build())) { + final Path dataFile = task.getDataPath(); + if (dataFile == null) { + utilGet(task, stream, true, -1, false); + } else { + try (FileUtils.CollocatedTempFile tempFile = FileUtils.newTempFile(dataFile)) { + task.setDataPath(tempFile.getPath(), false); + utilGet(task, stream, true, -1, false); + tempFile.move(); + } finally { + task.setDataPath(dataFile); + } + } + } + } + + @Override + protected void implPut(PutTask task) throws Exception { + ObjectName objectName = + objectNameMapper.apply(baseUri.relativize(task.getLocation()).getPath()); + task.getListener().transportStarted(0, task.getDataLength()); + final Path dataFile = task.getDataPath(); + if (dataFile == null) { + try (FileUtils.TempFile tempFile = FileUtils.newTempFile()) { + Files.copy(task.newInputStream(), tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING); + client.uploadObject(UploadObjectArgs.builder() + .bucket(objectName.getBucket()) + .object(objectName.getName()) + .filename(tempFile.getPath().toString()) + .build()); + } + } else { + client.uploadObject(UploadObjectArgs.builder() + .bucket(objectName.getBucket()) + .object(objectName.getName()) + .filename(dataFile.toString()) + .build()); + } + } + + @Override + protected void implClose() { + try { + client.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/MinioTransporterConfigurationKeys.java b/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/MinioTransporterConfigurationKeys.java new file mode 100644 index 000000000..c9cf2feec --- /dev/null +++ b/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/MinioTransporterConfigurationKeys.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.minio; + +import org.eclipse.aether.ConfigurationProperties; +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.transport.minio.internal.RepositoryIdObjectNameMapperFactory; + +/** + * Configuration for MinIO Transport. + * + * @since 2.0.2 + */ +public final class MinioTransporterConfigurationKeys { + private MinioTransporterConfigurationKeys() {} + + static final String CONFIG_PROPS_PREFIX = + ConfigurationProperties.PREFIX_TRANSPORT + MinioTransporterFactory.NAME + "."; + + /** + * Object name mapper to use. + * + * @configurationSource {@link RepositorySystemSession#getConfigProperties()} + * @configurationType {@link String} + * @configurationDefaultValue {@link #DEFAULT_OBJECT_NAME_MAPPER} + * @configurationRepoIdSuffix Yes + */ + public static final String CONFIG_PROP_OBJECT_NAME_MAPPER = CONFIG_PROPS_PREFIX + "objectNameMapper"; + + public static final String DEFAULT_OBJECT_NAME_MAPPER = RepositoryIdObjectNameMapperFactory.NAME; + + /** + * The fixed bucket name to use. + * + * @configurationSource {@link RepositorySystemSession#getConfigProperties()} + * @configurationType {@link String} + * @configurationDefaultValue {@link #DEFAULT_FIXED_BUCKET_NAME} + * @configurationRepoIdSuffix Yes + */ + public static final String CONFIG_PROP_FIXED_BUCKET_NAME = CONFIG_PROPS_PREFIX + "fixedBucketName"; + + public static final String DEFAULT_FIXED_BUCKET_NAME = "maven"; +} diff --git a/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/MinioTransporterFactory.java b/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/MinioTransporterFactory.java new file mode 100644 index 000000000..19e745e4e --- /dev/null +++ b/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/MinioTransporterFactory.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.minio; + +import javax.inject.Inject; +import javax.inject.Named; + +import java.util.Map; + +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.spi.connector.transport.Transporter; +import org.eclipse.aether.spi.connector.transport.TransporterFactory; +import org.eclipse.aether.transfer.NoTransporterException; +import org.eclipse.aether.util.ConfigUtils; + +import static java.util.Objects.requireNonNull; + +/** + * A transporter factory for repositories using the S3 API object storage using Minio. + * + * @since 2.0.2 + */ +@Named(MinioTransporterFactory.NAME) +public final class MinioTransporterFactory implements TransporterFactory { + public static final String NAME = "minio"; + + private static final float DEFAULT_PRIORITY = 0.0f; + + private float priority = DEFAULT_PRIORITY; + + private final Map objectNameMapperFactories; + + @Inject + public MinioTransporterFactory(Map objectNameMapperFactories) { + this.objectNameMapperFactories = requireNonNull(objectNameMapperFactories, "objectNameMapperFactories"); + } + + @Override + public float getPriority() { + return priority; + } + + public MinioTransporterFactory setPriority(float priority) { + this.priority = priority; + return this; + } + + @Override + public Transporter newInstance(RepositorySystemSession session, RemoteRepository repository) + throws NoTransporterException { + requireNonNull(session, "session cannot be null"); + requireNonNull(repository, "repository cannot be null"); + + // this check is here only to support "minio+http" and "s3+http" protocols by default. But also when + // raised priorities by uer, allow to "overtake" plain HTTP repositories, if needed. + RemoteRepository adjusted = repository; + if (DEFAULT_PRIORITY == priority) { + if ("minio+http".equalsIgnoreCase(repository.getProtocol()) + || "minio+https".equalsIgnoreCase(repository.getProtocol())) { + adjusted = new RemoteRepository.Builder(repository) + .setUrl(repository.getUrl().substring("minio+".length())) + .build(); + } else if ("s3+http".equalsIgnoreCase(repository.getProtocol()) + || "s3+https".equalsIgnoreCase(repository.getProtocol())) { + adjusted = new RemoteRepository.Builder(repository) + .setUrl(repository.getUrl().substring("s3+".length())) + .build(); + } else { + throw new NoTransporterException(repository); + } + } + String objectNameMapperConf = ConfigUtils.getString( + session, + MinioTransporterConfigurationKeys.DEFAULT_OBJECT_NAME_MAPPER, + MinioTransporterConfigurationKeys.CONFIG_PROP_OBJECT_NAME_MAPPER); + ObjectNameMapperFactory objectNameMapperFactory = objectNameMapperFactories.get(objectNameMapperConf); + if (objectNameMapperFactory == null) { + throw new NoTransporterException(repository, "Unknown object name mapper: " + objectNameMapperConf); + } + return new MinioTransporter(session, adjusted, objectNameMapperFactory.create(session, adjusted)); + } +} diff --git a/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/ObjectName.java b/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/ObjectName.java new file mode 100644 index 000000000..2369b7560 --- /dev/null +++ b/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/ObjectName.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.minio; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public final class ObjectName { + private final String bucket; + private final String name; + private final int hashCode; + + public ObjectName(String bucket, String name) { + this.bucket = requireNonNull(bucket); + this.name = requireNonNull(name); + + if (bucket.contains("/")) { + throw new IllegalArgumentException("invalid bucket name: " + bucket); + } + if (name.contains("\\")) { + throw new IllegalArgumentException("invalid object name: " + name); + } + + this.hashCode = Objects.hash(bucket, name); + } + + public String getBucket() { + return bucket; + } + + public String getName() { + return name; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ObjectName that = (ObjectName) o; + return Objects.equals(bucket, that.bucket) && Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return hashCode; + } + + @Override + public String toString() { + return bucket + "/" + name; + } + + public static String normalize(String name) { + return name.replace('\\', '/'); + } +} diff --git a/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/ObjectNameMapperFactory.java b/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/ObjectNameMapperFactory.java new file mode 100644 index 000000000..93f7ef123 --- /dev/null +++ b/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/ObjectNameMapperFactory.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.minio; + +import java.util.function.Function; + +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.repository.RemoteRepository; + +public interface ObjectNameMapperFactory { + Function create(RepositorySystemSession session, RemoteRepository repository); +} diff --git a/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/internal/FixedBucketObjectNameMapperFactory.java b/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/internal/FixedBucketObjectNameMapperFactory.java new file mode 100644 index 000000000..b695fd80a --- /dev/null +++ b/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/internal/FixedBucketObjectNameMapperFactory.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.minio.internal; + +import javax.inject.Named; +import javax.inject.Singleton; + +import java.util.function.Function; + +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.transport.minio.MinioTransporterConfigurationKeys; +import org.eclipse.aether.transport.minio.ObjectName; +import org.eclipse.aether.transport.minio.ObjectNameMapperFactory; +import org.eclipse.aether.util.ConfigUtils; + +/** + * A fixed bucket mapper, uses given bucket ID and then constructs object name using repository ID and layout path as + * object name. + */ +@Singleton +@Named(FixedBucketObjectNameMapperFactory.NAME) +public class FixedBucketObjectNameMapperFactory implements ObjectNameMapperFactory { + public static final String NAME = "fixedBucket"; + + @Override + public Function create(RepositorySystemSession session, RemoteRepository repository) { + String bucket = ConfigUtils.getString( + session, + MinioTransporterConfigurationKeys.CONFIG_PROP_FIXED_BUCKET_NAME, + MinioTransporterConfigurationKeys.CONFIG_PROP_FIXED_BUCKET_NAME + "." + repository.getId(), + MinioTransporterConfigurationKeys.DEFAULT_FIXED_BUCKET_NAME); + return path -> new ObjectName(bucket, repository.getId() + "/" + ObjectName.normalize(path)); + } +} diff --git a/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/internal/RepositoryIdObjectNameMapperFactory.java b/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/internal/RepositoryIdObjectNameMapperFactory.java new file mode 100644 index 000000000..8c0a4e891 --- /dev/null +++ b/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/internal/RepositoryIdObjectNameMapperFactory.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.minio.internal; + +import javax.inject.Named; +import javax.inject.Singleton; + +import java.util.function.Function; + +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.transport.minio.ObjectName; +import org.eclipse.aether.transport.minio.ObjectNameMapperFactory; + +/** + * A simple mapper, uses repository ID as bucket and layout path as + * object name. Assumes repository ID is a valid bucket name. + */ +@Singleton +@Named(RepositoryIdObjectNameMapperFactory.NAME) +public class RepositoryIdObjectNameMapperFactory implements ObjectNameMapperFactory { + public static final String NAME = "simple"; + + @Override + public Function create(RepositorySystemSession session, RemoteRepository repository) { + return path -> new ObjectName(repository.getId(), ObjectName.normalize(path)); + } +} diff --git a/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/package-info.java b/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/package-info.java new file mode 100644 index 000000000..247fede46 --- /dev/null +++ b/maven-resolver-transport-minio/src/main/java/org/eclipse/aether/transport/minio/package-info.java @@ -0,0 +1,29 @@ +// CHECKSTYLE_OFF: RegexpHeader +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Support for downloads/uploads via the S3 protocol. The implementation is backed by + * MinIO Java. + * The repository URL should be defined with protocol {@code minio+http} or {@code s3+http}. Note: use "https" if + * you are going for HTTPS remote, factory will merely strip "minio+" or "s3+" prefix assuming resulting URL will + * point to expected S3 endpoint. + * + * @since 2.0.0 + */ +package org.eclipse.aether.transport.minio; diff --git a/maven-resolver-transport-minio/src/test/java/org/eclipse/aether/transport/minio/MinioTransporterFactoryTest.java b/maven-resolver-transport-minio/src/test/java/org/eclipse/aether/transport/minio/MinioTransporterFactoryTest.java new file mode 100644 index 000000000..7ce8d2354 --- /dev/null +++ b/maven-resolver-transport-minio/src/test/java/org/eclipse/aether/transport/minio/MinioTransporterFactoryTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.minio; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import org.eclipse.aether.DefaultRepositorySystemSession; +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.spi.connector.transport.Transporter; +import org.eclipse.aether.transfer.NoTransporterException; +import org.eclipse.aether.transport.minio.internal.FixedBucketObjectNameMapperFactory; +import org.eclipse.aether.transport.minio.internal.RepositoryIdObjectNameMapperFactory; +import org.eclipse.aether.util.repository.AuthenticationBuilder; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * MinIO transporter UT. + */ +class MinioTransporterFactoryTest { + private Map factories; + private MinioTransporterFactory factory; + private RepositorySystemSession session; + private RemoteRepository repository; + + @BeforeEach + void startSuite() throws Exception { + Files.createDirectories(Paths.get(System.getProperty("java.io.tmpdir"))); // hack for Surefire + + factories = new HashMap<>(); + factories.put(RepositoryIdObjectNameMapperFactory.NAME, new RepositoryIdObjectNameMapperFactory()); + factories.put(FixedBucketObjectNameMapperFactory.NAME, new FixedBucketObjectNameMapperFactory()); + + factory = new MinioTransporterFactory(factories); + session = new DefaultRepositorySystemSession(h -> true); + repository = new RemoteRepository.Builder("repo", "default", "minio+http://localhost") + .setAuthentication(new AuthenticationBuilder() + .addUsername("username") + .addPassword("password") + .build()) + .build(); + } + + @AfterEach + public void stopSuite() { + // none + } + + @Test + void goodUrl() throws Exception { + try (Transporter transporter = factory.newInstance(session, repository)) { + // nope + } + } + + @Test + void wrongUrl() throws Exception { + final AtomicReference remoteRepository = new AtomicReference<>(); + + remoteRepository.set(new RemoteRepository.Builder("repo", "default", "http://localhost").build()); + assertThrows(NoTransporterException.class, () -> factory.newInstance(session, remoteRepository.get())); + remoteRepository.set(new RemoteRepository.Builder("repo", "default", "https://localhost").build()); + assertThrows(NoTransporterException.class, () -> factory.newInstance(session, remoteRepository.get())); + } + + @Test + void httpOvertake() throws Exception { + factory.setPriority(1000); + try (Transporter transporter = factory.newInstance( + session, + new RemoteRepository.Builder("repo", "default", "http://localhost") + .setAuthentication(new AuthenticationBuilder() + .addUsername("username") + .addPassword("password") + .build()) + .build())) { + // nope + } + } +} diff --git a/maven-resolver-transport-minio/src/test/java/org/eclipse/aether/transport/minio/MinioTransporterIT.java b/maven-resolver-transport-minio/src/test/java/org/eclipse/aether/transport/minio/MinioTransporterIT.java new file mode 100644 index 000000000..062d1acb4 --- /dev/null +++ b/maven-resolver-transport-minio/src/test/java/org/eclipse/aether/transport/minio/MinioTransporterIT.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.transport.minio; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; + +import io.minio.MakeBucketArgs; +import io.minio.MinioClient; +import io.minio.UploadObjectArgs; +import io.minio.errors.ErrorResponseException; +import org.eclipse.aether.DefaultRepositorySystemSession; +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.spi.connector.transport.GetTask; +import org.eclipse.aether.spi.connector.transport.PeekTask; +import org.eclipse.aether.spi.connector.transport.PutTask; +import org.eclipse.aether.spi.connector.transport.Transporter; +import org.eclipse.aether.transfer.NoTransporterException; +import org.eclipse.aether.transport.minio.internal.RepositoryIdObjectNameMapperFactory; +import org.eclipse.aether.util.FileUtils; +import org.eclipse.aether.util.repository.AuthenticationBuilder; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MinIOContainer; +import org.testcontainers.junit.jupiter.Testcontainers; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * MinIO transporter UT. + */ +@Testcontainers(disabledWithoutDocker = true) +class MinioTransporterIT { + private static final String BUCKET_NAME = "minio-repo"; + private static final String OBJECT_NAME = "dir/file.txt"; + private static final String OBJECT_CONTENT = "content"; + + private MinIOContainer minioContainer; + private MinioClient minioClient; + private RepositorySystemSession session; + private RemoteRepository repository; + private ObjectNameMapperFactory objectNameMapperFactory; + + @BeforeEach + void startSuite() throws Exception { + Files.createDirectories(Paths.get(System.getProperty("java.io.tmpdir"))); // hack for Surefire + + minioContainer = new MinIOContainer("minio/minio:latest"); + minioContainer.start(); + minioClient = MinioClient.builder() + .endpoint(minioContainer.getS3URL()) + .credentials(minioContainer.getUserName(), minioContainer.getPassword()) + .build(); + + minioClient.makeBucket(MakeBucketArgs.builder().bucket(BUCKET_NAME).build()); + try (FileUtils.TempFile tempFile = FileUtils.newTempFile()) { + Files.write(tempFile.getPath(), OBJECT_CONTENT.getBytes(StandardCharsets.UTF_8)); + minioClient.uploadObject(UploadObjectArgs.builder() + .bucket(BUCKET_NAME) + .object(OBJECT_NAME) + .filename(tempFile.getPath().toString()) + .build()); + } + + session = new DefaultRepositorySystemSession(h -> true); + repository = newRepo(RepositoryAuth.WITH); + objectNameMapperFactory = new RepositoryIdObjectNameMapperFactory(); + } + + @AfterEach + public void stopSuite() { + minioContainer.start(); + } + + enum RepositoryAuth { + WITHOUT, + WITH, + WRONG + } + + protected RemoteRepository newRepo(RepositoryAuth auth) { + RemoteRepository.Builder builder = + new RemoteRepository.Builder(BUCKET_NAME, "default", minioContainer.getS3URL()); + if (auth == RepositoryAuth.WITH) { + builder.setAuthentication(new AuthenticationBuilder() + .addUsername(minioContainer.getUserName()) + .addPassword(minioContainer.getPassword()) + .build()); + } else if (auth == RepositoryAuth.WRONG) { + builder.setAuthentication(new AuthenticationBuilder() + .addUsername("wrongusername") + .addPassword("wrongpassword") + .build()); + } + return builder.build(); + } + + @Test + void peekWithoutAuth() { + try { + new MinioTransporter( + session, newRepo(RepositoryAuth.WITHOUT), objectNameMapperFactory.create(session, repository)); + fail("Should throw"); + } catch (NoTransporterException e) { + assertTrue(e.getMessage().contains("No accessKey and/or secretKey provided")); + } + } + + @Test + void peekWithWrongAuth() throws Exception { + try (MinioTransporter transporter = new MinioTransporter( + session, newRepo(RepositoryAuth.WRONG), objectNameMapperFactory.create(session, repository))) { + try { + transporter.peek(new PeekTask(URI.create("test"))); + fail("Should throw"); + } catch (Exception e) { + assertInstanceOf(ErrorResponseException.class, e); + assertEquals(transporter.classify(e), Transporter.ERROR_OTHER); + } + } + } + + @Test + void peekNonexistent() throws Exception { + try (MinioTransporter transporter = new MinioTransporter( + session, newRepo(RepositoryAuth.WITH), objectNameMapperFactory.create(session, repository))) { + try { + transporter.peek(new PeekTask(URI.create("test"))); + fail("Should throw"); + } catch (Exception e) { + assertInstanceOf(ErrorResponseException.class, e); + assertEquals(transporter.classify(e), Transporter.ERROR_NOT_FOUND); + } + } + } + + @Test + void peekExistent() throws Exception { + try (MinioTransporter transporter = new MinioTransporter( + session, newRepo(RepositoryAuth.WITH), objectNameMapperFactory.create(session, repository))) { + try { + transporter.peek(new PeekTask(URI.create(OBJECT_NAME))); + } catch (Exception e) { + Assertions.fail("Should not throw"); + } + } + } + + @Test + void getNonexistent() throws Exception { + try (MinioTransporter transporter = new MinioTransporter( + session, newRepo(RepositoryAuth.WITH), objectNameMapperFactory.create(session, repository))) { + try { + transporter.get(new GetTask(URI.create("test"))); + fail("Should throw"); + } catch (Exception e) { + assertInstanceOf(ErrorResponseException.class, e); + assertEquals(transporter.classify(e), Transporter.ERROR_NOT_FOUND); + } + } + } + + @Test + void getExistent() throws Exception { + try (MinioTransporter transporter = new MinioTransporter( + session, newRepo(RepositoryAuth.WITH), objectNameMapperFactory.create(session, repository))) { + try { + GetTask task = new GetTask(URI.create(OBJECT_NAME)); + transporter.get(task); + assertEquals(OBJECT_CONTENT, new String(task.getDataBytes(), StandardCharsets.UTF_8)); + } catch (Exception e) { + Assertions.fail("Should not throw"); + } + } + } + + @Test + void putNonexistent() throws Exception { + try (MinioTransporter transporter = new MinioTransporter( + session, newRepo(RepositoryAuth.WITH), objectNameMapperFactory.create(session, repository))) { + try { + URI uri = URI.create("test"); + transporter.put(new PutTask(uri).setDataBytes(OBJECT_CONTENT.getBytes(StandardCharsets.UTF_8))); + GetTask task = new GetTask(uri); + transporter.get(task); + assertEquals(OBJECT_CONTENT, new String(task.getDataBytes(), StandardCharsets.UTF_8)); + } catch (Exception e) { + Assertions.fail("Should not throw"); + } + } + } + + @Test + void putExistent() throws Exception { + try (MinioTransporter transporter = new MinioTransporter( + session, newRepo(RepositoryAuth.WITH), objectNameMapperFactory.create(session, repository))) { + try { + URI uri = URI.create(OBJECT_NAME); + GetTask task = new GetTask(uri); + transporter.get(task); + assertEquals(OBJECT_CONTENT, new String(task.getDataBytes(), StandardCharsets.UTF_8)); + + String altContent = "altContent"; + transporter.put(new PutTask(uri).setDataBytes(altContent.getBytes(StandardCharsets.UTF_8))); + task = new GetTask(uri); + transporter.get(task); + assertEquals(altContent, new String(task.getDataBytes(), StandardCharsets.UTF_8)); + } catch (Exception e) { + Assertions.fail("Should not throw"); + } + } + } +} diff --git a/pom.xml b/pom.xml index 16eb7680c..2711da3d9 100644 --- a/pom.xml +++ b/pom.xml @@ -62,6 +62,7 @@ maven-resolver-transport-jdk-parent maven-resolver-transport-apache maven-resolver-transport-wagon + maven-resolver-transport-minio maven-resolver-generator-gnupg maven-resolver-supplier-mvn3 maven-resolver-supplier-mvn4 @@ -102,6 +103,7 @@ 0.9.0.M3 6.0.0 2.0.13 + 1.20.1 10.0.22 diff --git a/src/site/markdown/configuration.md b/src/site/markdown/configuration.md index c418b77b8..94d6856d1 100644 --- a/src/site/markdown/configuration.md +++ b/src/site/markdown/configuration.md @@ -132,17 +132,19 @@ under the License. | 105. | `"aether.transport.jdk.maxConcurrentRequests"` | `Integer` | The hard limit of maximum concurrent requests JDK transport can do. This is a workaround for the fact, that in HTTP/2 mode, JDK HttpClient initializes this value to Integer.MAX_VALUE (!) and lowers it on first response from the remote server (but it may be too late). See JDK bug JDK-8225647 for details. | `100` | 2.0.0 | Yes | Session Configuration | | 106. | `"aether.transport.jetty.followRedirects"` | `Boolean` | If enabled, Jetty client will follow HTTP redirects. | `true` | 2.0.1 | Yes | Session Configuration | | 107. | `"aether.transport.jetty.maxRedirects"` | `Integer` | The max redirect count to follow. | `5` | 2.0.1 | Yes | Session Configuration | -| 108. | `"aether.transport.wagon.config"` | `Object` | The configuration to use for the Wagon provider. | - | | Yes | Session Configuration | -| 109. | `"aether.transport.wagon.perms.dirMode"` | `String` | Octal numerical notation of permissions to set for newly created directories. Only considered by certain Wagon providers. | - | | Yes | Session Configuration | -| 110. | `"aether.transport.wagon.perms.fileMode"` | `String` | Octal numerical notation of permissions to set for newly created files. Only considered by certain Wagon providers. | - | | Yes | Session Configuration | -| 111. | `"aether.transport.wagon.perms.group"` | `String` | Group which should own newly created directories/files. Only considered by certain Wagon providers. | - | | Yes | Session Configuration | -| 112. | `"aether.trustedChecksumsSource.sparseDirectory"` | `Boolean` | Is checksum source enabled? | `false` | 1.9.0 | No | Session Configuration | -| 113. | `"aether.trustedChecksumsSource.sparseDirectory.basedir"` | `String` | The basedir where checksums are. If relative, is resolved from local repository root. | `".checksums"` | 1.9.0 | No | Session Configuration | -| 114. | `"aether.trustedChecksumsSource.sparseDirectory.originAware"` | `Boolean` | Is source origin aware? | `true` | 1.9.0 | No | Session Configuration | -| 115. | `"aether.trustedChecksumsSource.summaryFile"` | `Boolean` | Is checksum source enabled? | `false` | 1.9.0 | No | Session Configuration | -| 116. | `"aether.trustedChecksumsSource.summaryFile.basedir"` | `String` | The basedir where checksums are. If relative, is resolved from local repository root. | `".checksums"` | 1.9.0 | No | Session Configuration | -| 117. | `"aether.trustedChecksumsSource.summaryFile.originAware"` | `Boolean` | Is source origin aware? | `true` | 1.9.0 | No | Session Configuration | -| 118. | `"aether.updateCheckManager.sessionState"` | `String` | Manages the session state, i.e. influences if the same download requests to artifacts/metadata will happen multiple times within the same RepositorySystemSession. If "enabled" will enable the session state. If "bypass" will enable bypassing (i.e. store all artifact ids/metadata ids which have been updates but not evaluating those). All other values lead to disabling the session state completely. | `"enabled"` | | No | Session Configuration | +| 108. | `"aether.transport.minio.fixedBucketName"` | `String` | The fixed bucket name to use. | `"maven"` | 2.0.2 | Yes | Session Configuration | +| 109. | `"aether.transport.minio.objectNameMapper"` | `String` | Object name mapper to use. | `"simple"` | 2.0.2 | Yes | Session Configuration | +| 110. | `"aether.transport.wagon.config"` | `Object` | The configuration to use for the Wagon provider. | - | | Yes | Session Configuration | +| 111. | `"aether.transport.wagon.perms.dirMode"` | `String` | Octal numerical notation of permissions to set for newly created directories. Only considered by certain Wagon providers. | - | | Yes | Session Configuration | +| 112. | `"aether.transport.wagon.perms.fileMode"` | `String` | Octal numerical notation of permissions to set for newly created files. Only considered by certain Wagon providers. | - | | Yes | Session Configuration | +| 113. | `"aether.transport.wagon.perms.group"` | `String` | Group which should own newly created directories/files. Only considered by certain Wagon providers. | - | | Yes | Session Configuration | +| 114. | `"aether.trustedChecksumsSource.sparseDirectory"` | `Boolean` | Is checksum source enabled? | `false` | 1.9.0 | No | Session Configuration | +| 115. | `"aether.trustedChecksumsSource.sparseDirectory.basedir"` | `String` | The basedir where checksums are. If relative, is resolved from local repository root. | `".checksums"` | 1.9.0 | No | Session Configuration | +| 116. | `"aether.trustedChecksumsSource.sparseDirectory.originAware"` | `Boolean` | Is source origin aware? | `true` | 1.9.0 | No | Session Configuration | +| 117. | `"aether.trustedChecksumsSource.summaryFile"` | `Boolean` | Is checksum source enabled? | `false` | 1.9.0 | No | Session Configuration | +| 118. | `"aether.trustedChecksumsSource.summaryFile.basedir"` | `String` | The basedir where checksums are. If relative, is resolved from local repository root. | `".checksums"` | 1.9.0 | No | Session Configuration | +| 119. | `"aether.trustedChecksumsSource.summaryFile.originAware"` | `Boolean` | Is source origin aware? | `true` | 1.9.0 | No | Session Configuration | +| 120. | `"aether.updateCheckManager.sessionState"` | `String` | Manages the session state, i.e. influences if the same download requests to artifacts/metadata will happen multiple times within the same RepositorySystemSession. If "enabled" will enable the session state. If "bypass" will enable bypassing (i.e. store all artifact ids/metadata ids which have been updates but not evaluating those). All other values lead to disabling the session state completely. | `"enabled"` | | No | Session Configuration | All properties which have `yes` in the column `Supports Repo ID Suffix` can be optionally configured specifically for a repository id. In that case the configuration property needs to be suffixed with a period followed by the repository id of the repository to configure, e.g. `aether.connector.http.headers.central` for repository with id `central`.