From f6668bae4702583918ba7e089d06cee09cc7e126 Mon Sep 17 00:00:00 2001
From: Enno Ruijters <eruijters@betterbe.com>
Date: Thu, 28 Sep 2023 13:45:14 +0200
Subject: [PATCH] feat: Support filtered DirectoryStream (#381)

* Add support for filtered DirectoryStreams.
* Add unit test

---------

Co-authored-by: Marcel Hekman <mhekman@betterbe.com>
---
 .../storage/s3fs/S3FileSystemProvider.java    |  2 +-
 .../storage/s3fs/S3FilteredIterator.java      | 84 +++++++++++++++++++
 .../NewDirectoryStreamTest.java               | 57 ++++++++++---
 3 files changed, 131 insertions(+), 12 deletions(-)
 create mode 100644 src/main/java/org/carlspring/cloud/storage/s3fs/S3FilteredIterator.java

diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java
index 3f81b341..03d3a135 100644
--- a/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java
+++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java
@@ -512,7 +512,7 @@ public void close()
             @Override
             public Iterator<Path> iterator()
             {
-                return new S3Iterator(s3Path);
+                return new S3FilteredIterator(s3Path, filter);
             }
         };
     }
diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3FilteredIterator.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3FilteredIterator.java
new file mode 100644
index 00000000..3e8968fc
--- /dev/null
+++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3FilteredIterator.java
@@ -0,0 +1,84 @@
+package org.carlspring.cloud.storage.s3fs;
+
+import java.io.IOException;
+import java.nio.file.DirectoryStream.Filter;
+import java.nio.file.Path;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+public class S3FilteredIterator implements Iterator<Path>
+{
+
+    private final S3Iterator s3iterator;
+
+    private final Filter<? super Path> filter;
+
+    private Path cursor;
+
+    private boolean cursorIsCurrent;
+
+    public S3FilteredIterator(S3Path s3Path,
+                              Filter<? super Path> filter)
+    {
+        this.filter = filter;
+        this.s3iterator = new S3Iterator(s3Path);
+        clearCursor();
+    }
+
+    @Override
+    public boolean hasNext()
+    {
+        if (!cursorIsCurrent)
+        {
+            findNextFiltered();
+        }
+        return cursorIsCurrent;
+    }
+
+    @Override
+    public Path next()
+    {
+        if (!hasNext())
+        {
+            throw new NoSuchElementException();
+        }
+        Path next = cursor;
+        clearCursor();
+        return next;
+    }
+
+    private void findNextFiltered()
+    {
+        try
+        {
+            while (s3iterator.hasNext())
+            {
+                S3Path next = s3iterator.next();
+                if (filter.accept(next))
+                {
+                    cursor = next;
+                    cursorIsCurrent = true;
+                    return;
+                }
+            }
+            clearCursor();
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void clearCursor()
+    {
+        cursor = null;
+        cursorIsCurrent = false;
+    }
+
+    @Override
+    public void remove()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+}
diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewDirectoryStreamTest.java b/src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewDirectoryStreamTest.java
index 5ee2073f..c75db741 100644
--- a/src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewDirectoryStreamTest.java
+++ b/src/test/java/org/carlspring/cloud/storage/s3fs/fileSystemProvider/NewDirectoryStreamTest.java
@@ -104,6 +104,35 @@ void removeIteratorStreamDirectoryReader()
         }
     }
 
+    @Test
+    public void directoryStreamWithFilter()
+        throws IOException
+    {
+        // fixtures
+        S3ClientMock client = S3MockFactory.getS3ClientMock();
+        client.bucket("bucketA")
+              .dir("dir1") //
+              .file("dir/file1.txt", "content".getBytes()) //
+              .file("dir/file2.sql", "content".getBytes()) //
+              .file("dir/file3.txt", "content".getBytes()) //
+              .file("dir/file4.sql", "content".getBytes()) //
+              .file("dir/tmp_file.txt", "content".getBytes());
+
+        // act
+        Path base = createNewS3FileSystem().getPath("/bucketA", "dir");
+        DirectoryStream.Filter<Path> filter = entry -> {
+            String filename = entry.getFileName().toString();
+            return filename.startsWith("tmp_") || filename.endsWith(".sql");
+        };
+
+        // act
+        try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(base, filter))
+        {
+            // assert
+            assertDirectoryStream(dirStream, "file2.sql", "file4.sql", "tmp_file.txt");
+        }
+    }
+
     @Test
     void list999Paths()
             throws IOException
@@ -146,21 +175,27 @@ private void assertNewDirectoryStream(Path base,
     {
         try (DirectoryStream<Path> dir = Files.newDirectoryStream(base))
         {
-            assertNotNull(dir);
-            assertNotNull(dir.iterator());
-            assertTrue(dir.iterator().hasNext());
+            assertDirectoryStream(dir, files);
+        }
+    }
 
-            Set<String> filesNamesExpected = new HashSet<>(Arrays.asList(files));
-            Set<String> filesNamesActual = new HashSet<>();
+    private void assertDirectoryStream(DirectoryStream<Path> dir,
+                                       final String... files)
+    {
+        assertNotNull(dir);
+        assertNotNull(dir.iterator());
+        assertTrue(dir.iterator().hasNext());
 
-            for (Path path : dir)
-            {
-                String fileName = path.getFileName().toString();
-                filesNamesActual.add(fileName);
-            }
+        Set<String> filesNamesExpected = new HashSet<>(Arrays.asList(files));
+        Set<String> filesNamesActual = new HashSet<>();
 
-            assertEquals(filesNamesExpected, filesNamesActual);
+        for (Path path : dir)
+        {
+            String fileName = path.getFileName().toString();
+            filesNamesActual.add(fileName);
         }
+
+        assertEquals(filesNamesExpected, filesNamesActual);
     }
 
     /**