From e03a486486b26a73aff27c5b76581efdb579d4e4 Mon Sep 17 00:00:00 2001 From: Steve Todorov Date: Thu, 7 Nov 2024 16:49:21 +0200 Subject: [PATCH] feat!: Improve file attribute caching (#854) Set the default cache size to 30k --- src/main/java/org/carlspring/cloud/storage/s3fs/S3Factory.java | 2 +- .../cloud/storage/s3fs/fileSystemProvider/CacheTestIT.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3Factory.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3Factory.java index e3f1ea1b..7836e8c5 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/S3Factory.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3Factory.java @@ -53,7 +53,7 @@ public abstract class S3Factory * Total size of {@link S3BasicFileAttributes} and {@link S3PosixFileAttributes} cache. */ public static final String CACHE_ATTRIBUTES_SIZE = "s3fs.cache.attributes.size"; - public static final int CACHE_ATTRIBUTES_SIZE_DEFAULT = 5000; + public static final int CACHE_ATTRIBUTES_SIZE_DEFAULT = 30000; public static final String REQUEST_METRIC_COLLECTOR_CLASS = "s3fs.request.metric.collector.class"; diff --git a/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/CacheTestIT.java b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/CacheTestIT.java index 56d13387..4fde71a5 100644 --- a/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/CacheTestIT.java +++ b/src/testIntegration/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/CacheTestIT.java @@ -224,6 +224,8 @@ void testScenario002() throws IOException long startTime = System.currentTimeMillis(); AtomicInteger processedFileCount = new AtomicInteger(0); + // Reminder: The `.limit()` will not limit the amount of requests sent to AWS. + // If the rootPath contains 100k files and the `.limit(1000)` -- it will still fetch the 100k files first. try (Stream stream = Files.list(rootPath).limit(totalFilesAdded.get())) { stream.filter(path -> path != null &&