Skip to content

Commit

Permalink
feat: Allow configuring cacheControl via s3fs.request.header.cache-co…
Browse files Browse the repository at this point in the history
…ntrol flag (#711)
  • Loading branch information
steve-todorov authored May 30, 2023
1 parent 73fbca4 commit f1cb117
Show file tree
Hide file tree
Showing 13 changed files with 349 additions and 94 deletions.
1 change: 1 addition & 0 deletions build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ dependencies {
testImplementation("org.mockito:mockito-junit-jupiter:3.9.0")
testImplementation("org.testcontainers:testcontainers:1.18.1")
testImplementation("org.testcontainers:testcontainers:1.18.1")
testImplementation("org.assertj:assertj-core:3.24.2")
}

configure<com.adarshr.gradle.testlogger.TestLoggerExtension> {
Expand Down
47 changes: 24 additions & 23 deletions docs/content/reference/configuration-options.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,26 +4,27 @@

A complete list of environment variables which can be set to configure the client.

| Key | Default | Description |
| ------------------------------------------| ----------- |---------------------------------------------------------------------------------------------------------------------------------------- |
| s3fs.access.key | none | <small>AWS access key, used to identify the user interacting with AWS</small> |
| s3fs.secret.key | none | <small>AWS secret access key, used to authenticate the user interacting with AWS</small> |
| s3fs.request.metric.collector.class | TODO | <small>Fully-qualified class name to instantiate an AWS SDK request/response metric collector</small> |
| s3fs.connection.timeout | TODO | <small>Timeout (in milliseconds) for establishing a connection to a remote service</small> |
| s3fs.max.connections | TODO | <small>Maximum number of connections allowed in a connection pool</small> |
| s3fs.max.retry.error | TODO | <small>Maximum number of times that a single request should be retried, assuming it fails for a retryable error</small> |
| s3fs.protocol | TODO | <small>Protocol (HTTP or HTTPS) to use when connecting to AWS</small> |
| s3fs.proxy.domain | none | <small>For NTLM proxies: The Windows domain name to use when authenticating with the proxy</small> |
| s3fs.proxy.host | none | <small>Proxy host name either from the configured endpoint or from the "http.proxyHost" system property</small> |
| s3fs.proxy.password | none | <small>The password to use when connecting through a proxy</small> |
| s3fs.proxy.port | none | <small>Proxy port either from the configured endpoint or from the "http.proxyPort" system property</small> |
| s3fs.proxy.username | none | <small>The username to use when connecting through a proxy</small> |
| s3fs.proxy.workstation | none | <small>For NTLM proxies: The Windows workstation name to use when authenticating with the proxy</small> |
| s3fs.region | none | <small>The AWS Region to configure the client</small> |
| s3fs.socket.send.buffer.size.hint | TODO | <small>The size hint (in bytes) for the low level TCP send buffer</small> |
| s3fs.socket.receive.buffer.size.hint | TODO | <small>The size hint (in bytes) for the low level TCP receive buffer</small> |
| s3fs.socket.timeout | TODO | <small>Timeout (in milliseconds) for each read to the underlying socket</small> |
| s3fs.user.agent.prefix | TODO | <small>Prefix of the user agent that is sent with each request to AWS</small> |
| s3fs.amazon.s3.factory.class | TODO | <small>Fully-qualified class name to instantiate a S3 factory base class which creates a S3 client instance</small> |
| s3fs.signer.override | TODO | <small>Fully-qualified class name to define the signer that should be used when authenticating with AWS</small> |
| s3fs.path.style.access | TODO | <small>Boolean that indicates whether the client uses path-style access for all requests</small> |
| Key | Default | Description |
| ------------------------------------------|---------|-------------------------------------------------------------------------------------------------------------------------|
| s3fs.access.key | none | <small>AWS access key, used to identify the user interacting with AWS</small> |
| s3fs.secret.key | none | <small>AWS secret access key, used to authenticate the user interacting with AWS</small> |
| s3fs.request.metric.collector.class | TODO | <small>Fully-qualified class name to instantiate an AWS SDK request/response metric collector</small> |
| s3fs.connection.timeout | TODO | <small>Timeout (in milliseconds) for establishing a connection to a remote service</small> |
| s3fs.max.connections | TODO | <small>Maximum number of connections allowed in a connection pool</small> |
| s3fs.max.retry.error | TODO | <small>Maximum number of times that a single request should be retried, assuming it fails for a retryable error</small> |
| s3fs.protocol | TODO | <small>Protocol (HTTP or HTTPS) to use when connecting to AWS</small> |
| s3fs.proxy.domain | none | <small>For NTLM proxies: The Windows domain name to use when authenticating with the proxy</small> |
| s3fs.proxy.host | none | <small>Proxy host name either from the configured endpoint or from the "http.proxyHost" system property</small> |
| s3fs.proxy.password | none | <small>The password to use when connecting through a proxy</small> |
| s3fs.proxy.port | none | <small>Proxy port either from the configured endpoint or from the "http.proxyPort" system property</small> |
| s3fs.proxy.username | none | <small>The username to use when connecting through a proxy</small> |
| s3fs.proxy.workstation | none | <small>For NTLM proxies: The Windows workstation name to use when authenticating with the proxy</small> |
| s3fs.region | none | <small>The AWS Region to configure the client</small> |
| s3fs.socket.send.buffer.size.hint | TODO | <small>The size hint (in bytes) for the low level TCP send buffer</small> |
| s3fs.socket.receive.buffer.size.hint | TODO | <small>The size hint (in bytes) for the low level TCP receive buffer</small> |
| s3fs.socket.timeout | TODO | <small>Timeout (in milliseconds) for each read to the underlying socket</small> |
| s3fs.user.agent.prefix | TODO | <small>Prefix of the user agent that is sent with each request to AWS</small> |
| s3fs.amazon.s3.factory.class | TODO | <small>Fully-qualified class name to instantiate a S3 factory base class which creates a S3 client instance</small> |
| s3fs.signer.override | TODO | <small>Fully-qualified class name to define the signer that should be used when authenticating with AWS</small> |
| s3fs.path.style.access | TODO | <small>Boolean that indicates whether the client uses path-style access for all requests</small> |
| s3fs.request.header.cache-control | blank | <small>Configures the `cacheControl` on request builders (i.e. `CopyObjectRequest`, `PutObjectRequest`, etc) |
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import software.amazon.awssdk.http.apache.ProxyConfiguration;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.regions.providers.AwsRegionProvider;
import software.amazon.awssdk.regions.providers.AwsRegionProviderChain;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3ClientBuilder;
import software.amazon.awssdk.services.s3.S3Configuration;
Expand Down Expand Up @@ -84,6 +83,8 @@ public abstract class S3Factory

public static final String PATH_STYLE_ACCESS = "s3fs.path.style.access";

public static final String REQUEST_HEADER_CACHE_CONTROL = "s3fs.request.header.cache-control";

private static final Logger LOGGER = LoggerFactory.getLogger(S3Factory.class);

private static final String DEFAULT_PROTOCOL = Protocol.HTTPS.toString();
Expand Down
17 changes: 13 additions & 4 deletions src/main/java/org/carlspring/cloud/storage/s3fs/S3FileChannel.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,7 @@
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
Expand Down Expand Up @@ -67,6 +65,15 @@ public class S3FileChannel
*/
private final Lock writeReadChannelLock = readWriteLock.readLock();

public S3FileChannel(final S3Path path,
final Set<? extends OpenOption> options,
final ExecutorService executor,
final boolean tempFileRequired)
throws IOException
{
this(path, options, executor, tempFileRequired, new HashMap<>());
}

/**
* Open or creates a file, returning a file channel.
*
Expand All @@ -79,13 +86,15 @@ public class S3FileChannel
public S3FileChannel(final S3Path path,
final Set<? extends OpenOption> options,
final ExecutorService executor,
final boolean tempFileRequired)
final boolean tempFileRequired,
final Map<String, String> properties)
throws IOException
{
openCloseLock.lock();

this.path = path;
this.options = Collections.unmodifiableSet(new HashSet<>(options));
String headerCacheControlProperty = path.getFileSystem().getRequestHeaderCacheControlProperty();
boolean exists = path.getFileSystem().provider().exists(path);
boolean removeTempFile = false;

Expand Down
33 changes: 28 additions & 5 deletions src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystem.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import java.nio.file.PathMatcher;
import java.nio.file.WatchService;
import java.nio.file.attribute.UserPrincipalLookupService;
import java.util.Properties;
import java.util.Set;

import com.google.common.collect.ImmutableList;
Expand Down Expand Up @@ -35,16 +36,28 @@ public class S3FileSystem

private final int cache;

private final Properties properties;

public S3FileSystem(final S3FileSystemProvider provider,
final String key,
final S3Client client,
final String endpoint)
final String endpoint,
Properties properties)
{
this.provider = provider;
this.key = key;
this.client = client;
this.endpoint = endpoint;
this.cache = 60000; // 1 minute cache for the s3Path
this.properties = properties;
}

public S3FileSystem(final S3FileSystemProvider provider,
final String key,
final S3Client client,
final String endpoint)
{
this(provider, key, client, endpoint, new Properties());
}

@Override
Expand Down Expand Up @@ -172,6 +185,20 @@ public String[] key2Parts(String keyParts)
return split;
}

public int getCache()
{
return cache;
}


/**
* @return The value of the {@link S3Factory#REQUEST_HEADER_CACHE_CONTROL} property. Default is empty.
*/
public String getRequestHeaderCacheControlProperty()
{
return properties.getProperty(S3Factory.REQUEST_HEADER_CACHE_CONTROL, ""); // default is nothing.
}

@Override
public int hashCode()
{
Expand Down Expand Up @@ -230,8 +257,4 @@ public int compareTo(final S3FileSystem o)
return key.compareTo(o.getKey());
}

public int getCache()
{
return cache;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -597,7 +597,10 @@ public OutputStream newOutputStream(final Path path,


final Map<String, String> metadata = buildMetadataFromPath(path);
return new S3OutputStream(s3Path.getFileSystem().getClient(), s3Path.toS3ObjectId(), metadata);

S3FileSystem fileSystem = s3Path.getFileSystem();

return new S3OutputStream(fileSystem.getClient(), s3Path.toS3ObjectId(), null, metadata, fileSystem.getRequestHeaderCacheControlProperty());
}

private void validateCreateAndTruncateOptions(final Path path,
Expand Down Expand Up @@ -696,6 +699,7 @@ public void createDirectory(Path dir,
final PutObjectRequest request = PutObjectRequest.builder()
.bucket(bucketName)
.key(directoryKey)
.cacheControl(s3Path.getFileSystem().getRequestHeaderCacheControlProperty())
.contentLength(0L)
.build();

Expand Down Expand Up @@ -869,6 +873,7 @@ public void copy(Path source,

final CopyObjectRequest request = CopyObjectRequest.builder()
.copySource(encodedUrl)
.cacheControl(s3Target.getFileSystem().getRequestHeaderCacheControlProperty())
.destinationBucket(bucketNameTarget)
.destinationKey(keyTarget)
.build();
Expand Down Expand Up @@ -1097,7 +1102,8 @@ public S3FileSystem createFileSystem(URI uri,
final String key = getFileSystemKey(uri, props);
final S3Client client = getS3Client(uri, props);
final String host = uri.getHost();
return new S3FileSystem(this, key, client, host);
final Properties properties = new Properties(props);
return new S3FileSystem(this, key, client, host, properties);
}

protected S3Client getS3Client(URI uri,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,8 @@ public final class S3OutputStream
*/
private List<String> partETags;

private final String requestCacheControlHeader;

/**
* Creates a new {@code S3OutputStream} that writes data directly into the S3 object with the given {@code objectId}.
* No special object metadata or storage class will be attached to the object.
Expand All @@ -115,6 +117,7 @@ public S3OutputStream(final S3Client s3Client,
this.objectId = requireNonNull(objectId);
this.metadata = new HashMap<>();
this.storageClass = null;
this.requestCacheControlHeader = "";
}

/**
Expand All @@ -132,8 +135,9 @@ public S3OutputStream(final S3Client s3Client,
{
this.s3Client = requireNonNull(s3Client);
this.objectId = requireNonNull(objectId);
this.storageClass = storageClass;
this.metadata = new HashMap<>();
this.storageClass = storageClass;
this.requestCacheControlHeader = "";
}

/**
Expand All @@ -154,6 +158,7 @@ public S3OutputStream(final S3Client s3Client,
this.objectId = requireNonNull(objectId);
this.storageClass = null;
this.metadata = new HashMap<>(metadata);
this.requestCacheControlHeader = "";
}

/**
Expand All @@ -175,6 +180,31 @@ public S3OutputStream(final S3Client s3Client,
this.objectId = requireNonNull(objectId);
this.storageClass = storageClass;
this.metadata = new HashMap<>(metadata);
this.requestCacheControlHeader = "";
}

/**
* Creates a new {@code S3OutputStream} that writes data directly into the S3 object with the given {@code objectId}.
* The given {@code metadata} will be attached to the written object.
*
* @param s3Client S3 ClientAPI to use
* @param objectId ID of the S3 object to store data into
* @param storageClass S3 Client storage class to apply to the newly created S3 object, if any
* @param metadata metadata to attach to the written object
* @param requestCacheControlHeader Controls
* @throws NullPointerException if at least one parameter except {@code storageClass} is {@code null}
*/
public S3OutputStream(final S3Client s3Client,
final S3ObjectId objectId,
final StorageClass storageClass,
final Map<String, String> metadata,
final String requestCacheControlHeader)
{
this.s3Client = requireNonNull(s3Client);
this.objectId = requireNonNull(objectId);
this.storageClass = storageClass;
this.metadata = new HashMap<>(metadata);
this.requestCacheControlHeader = requestCacheControlHeader;
}

//protected for testing purposes
Expand Down Expand Up @@ -435,6 +465,7 @@ private void putObject(final long contentLength,
final PutObjectRequest.Builder requestBuilder = PutObjectRequest.builder()
.bucket(objectId.getBucket())
.key(objectId.getKey())
.cacheControl(requestCacheControlHeader)
.contentLength(contentLength)
.contentType(contentType)
.metadata(metadataMap);
Expand Down
Loading

0 comments on commit f1cb117

Please sign in to comment.