Merge pull request #3989 from wasmerio/fixes-for-overlay-fs

OverlayFS now has COW and some limited unit tests

lib/virtual-fs/Cargo.toml

@@ -12,6 +12,7 @@ rust-version.workspace = true
 [dependencies]
 libc = { version = "^0.2", default-features = false, optional = true }
 thiserror = "1"
+futures = { version = "0.3" }
 tracing = { version = "0.1" }
 typetag = { version = "0.1", optional = true }
 webc = { version = "5.0", optional = true }

lib/virtual-fs/src/arc_box_file.rs

@@ -1,16 +1,19 @@
 //! Used for sharing references to the same file across multiple file systems,
 //! effectively this is a symbolic link without all the complex path redirection
 
-use crate::VirtualFile;
-use derivative::Derivative;
-use std::pin::Pin;
-use std::task::{Context, Poll};
 use std::{
     io::{self, *},
+    pin::Pin,
     sync::{Arc, Mutex},
+    task::{Context, Poll},
 };
+
+use derivative::Derivative;
+use futures::future::BoxFuture;
 use tokio::io::{AsyncRead, AsyncSeek, AsyncWrite};
 
+use crate::VirtualFile;
+
 #[derive(Derivative, Clone)]
 #[derivative(Debug)]
 pub struct ArcBoxFile {
@@ -108,9 +111,11 @@ impl VirtualFile for ArcBoxFile {
         let mut inner = self.inner.lock().unwrap();
         inner.set_len(new_size)
     }
-    fn unlink(&mut self) -> crate::Result<()> {
+    fn unlink(&mut self) -> BoxFuture<'static, crate::Result<()>> {
         let mut inner = self.inner.lock().unwrap();
-        inner.unlink()
+        let fut = inner.unlink();
+        drop(inner);
+        Box::pin(async { fut.await })
     }
     fn is_open(&self) -> bool {
         let inner = self.inner.lock().unwrap();

lib/virtual-fs/src/arc_file.rs

@@ -3,6 +3,7 @@
 
 use crate::{ClonableVirtualFile, VirtualFile};
 use derivative::Derivative;
+use futures::future::BoxFuture;
 use std::pin::Pin;
 use std::task::{Context, Poll};
 use std::{
@@ -126,9 +127,11 @@ where
         let mut inner = self.inner.lock().unwrap();
         inner.set_len(new_size)
     }
-    fn unlink(&mut self) -> crate::Result<()> {
+    fn unlink(&mut self) -> BoxFuture<'static, crate::Result<()>> {
         let mut inner = self.inner.lock().unwrap();
-        inner.unlink()
+        let fut = inner.unlink();
+        drop(inner);
+        Box::pin(async move { fut.await })
     }
     fn is_open(&self) -> bool {
         let inner = self.inner.lock().unwrap();

lib/virtual-fs/src/arc_fs.rs

@@ -2,10 +2,7 @@
 //! can pass clonable file systems with a `Box<dyn FileSystem>` to other
 //! interfaces
 
-use std::path::Path;
-use std::sync::Arc;
-#[allow(unused_imports, dead_code)]
-use tracing::{debug, error, info, trace, warn};
+use std::{path::Path, sync::Arc};
 
 use crate::*;
 
@@ -33,8 +30,8 @@ impl FileSystem for ArcFileSystem {
         self.fs.remove_dir(path)
     }
 
-    fn rename(&self, from: &Path, to: &Path) -> Result<()> {
-        self.fs.rename(from, to)
+    fn rename<'a>(&'a self, from: &'a Path, to: &'a Path) -> BoxFuture<'a, Result<()>> {
+        Box::pin(async { self.fs.rename(from, to).await })
     }
 
     fn metadata(&self, path: &Path) -> Result<Metadata> {

lib/virtual-fs/src/buffer_file.rs

@@ -5,6 +5,7 @@ use std::io::{self, *};
 use std::pin::Pin;
 use std::task::{Context, Poll};
 
+use futures::future::BoxFuture;
 use tokio::io::{AsyncRead, AsyncSeek, AsyncWrite};
 
 use crate::VirtualFile;
@@ -84,8 +85,8 @@ impl VirtualFile for BufferFile {
         self.data.get_mut().resize(new_size as usize, 0);
         Ok(())
     }
-    fn unlink(&mut self) -> crate::Result<()> {
-        Ok(())
+    fn unlink(&mut self) -> BoxFuture<'static, crate::Result<()>> {
+        Box::pin(async { Ok(()) })
     }
     fn poll_read_ready(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<usize>> {
         let cur = self.data.stream_position().unwrap_or_default();

lib/virtual-fs/src/combine_file.rs

@@ -41,8 +41,9 @@ impl VirtualFile for CombineFile {
         self.tx.set_len(new_size)
     }
 
-    fn unlink(&mut self) -> Result<()> {
-        self.tx.unlink()
+    fn unlink(&mut self) -> BoxFuture<'static, Result<()>> {
+        let fut = self.tx.unlink();
+        Box::pin(async { fut.await })
     }
 
     fn poll_read_ready(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<usize>> {

lib/virtual-fs/src/cow_file.rs

@@ -1,6 +1,7 @@
 //! Used for /dev/zero - infinitely returns zero
 //! which is useful for commands like `dd if=/dev/zero of=bigfile.img size=1G`
 
+use futures::future::BoxFuture;
 use replace_with::replace_with_or_abort;
 use std::io::{self, *};
 use std::pin::Pin;
@@ -204,9 +205,9 @@ impl VirtualFile for CopyOnWriteFile {
     fn set_len(&mut self, new_size: u64) -> crate::Result<()> {
         self.buf.set_len(new_size)
     }
-    fn unlink(&mut self) -> crate::Result<()> {
-        self.buf.set_len(0)?;
-        Ok(())
+    fn unlink(&mut self) -> BoxFuture<'static, crate::Result<()>> {
+        let ret = self.buf.set_len(0);
+        Box::pin(async move { ret })
     }
     fn poll_read_ready(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<usize>> {
         match self.poll_copy_progress(cx) {

lib/virtual-fs/src/dual_write_file.rs

@@ -45,12 +45,13 @@ impl VirtualFile for DualWriteFile {
         self.inner.size()
     }
 
-    fn set_len(&mut self, new_size: u64) -> Result<()> {
+    fn set_len(&mut self, new_size: u64) -> crate::Result<()> {
         self.inner.set_len(new_size)
     }
 
-    fn unlink(&mut self) -> Result<()> {
-        self.inner.unlink()
+    fn unlink(&mut self) -> BoxFuture<'static, Result<()>> {
+        let fut = self.inner.unlink();
+        Box::pin(async { fut.await })
     }
 
     fn poll_read_ready(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<usize>> {

lib/virtual-fs/src/empty_fs.rs

@@ -24,8 +24,8 @@ impl FileSystem for EmptyFileSystem {
         Err(FsError::EntryNotFound)
     }
 
-    fn rename(&self, from: &Path, to: &Path) -> Result<()> {
-        Err(FsError::EntryNotFound)
+    fn rename<'a>(&'a self, from: &'a Path, to: &'a Path) -> BoxFuture<'a, Result<()>> {
+        Box::pin(async { Err(FsError::EntryNotFound) })
     }
 
     fn metadata(&self, path: &Path) -> Result<Metadata> {

lib/virtual-fs/src/filesystems.rs

@@ -4,7 +4,7 @@ use crate::FileSystem;
 pub trait FileSystems<'a>: 'a {
     // FIXME(Michael-F-Bryan): Rewrite this to use GATs when we bump the MSRV to
     // 1.65 or higher. That'll get rid of all the lifetimes and HRTBs.
-    type Iter: IntoIterator<Item = &'a dyn FileSystem> + 'a;
+    type Iter: IntoIterator<Item = &'a (dyn FileSystem + Send)> + Send + 'a;
 
     /// Get something that can be used to iterate over the underlying
     /// filesystems.
@@ -25,7 +25,7 @@ where
 
 impl<'a, T> FileSystems<'a> for Vec<T>
 where
-    T: FileSystem,
+    T: FileSystem + Send,
 {
     type Iter = <[T] as FileSystems<'a>>::Iter;
 
@@ -36,15 +36,15 @@ where
 
 impl<'a, T, const N: usize> FileSystems<'a> for [T; N]
 where
-    T: FileSystem,
+    T: FileSystem + Send,
 {
-    type Iter = [&'a dyn FileSystem; N];
+    type Iter = [&'a (dyn FileSystem + Send); N];
 
     fn filesystems(&'a self) -> Self::Iter {
         // TODO: rewrite this when array::each_ref() is stable
         let mut i = 0;
         [(); N].map(|_| {
-            let f = &self[i] as &dyn FileSystem;
+            let f = &self[i] as &(dyn FileSystem + Send);
             i += 1;
             f
         })
@@ -53,17 +53,17 @@ where
 
 impl<'a, T> FileSystems<'a> for [T]
 where
-    T: FileSystem,
+    T: FileSystem + Send,
 {
-    type Iter = std::iter::Map<std::slice::Iter<'a, T>, fn(&T) -> &dyn FileSystem>;
+    type Iter = std::iter::Map<std::slice::Iter<'a, T>, fn(&T) -> &(dyn FileSystem + Send)>;
 
     fn filesystems(&'a self) -> Self::Iter {
-        self.iter().map(|fs| fs as &dyn FileSystem)
+        self.iter().map(|fs| fs as &(dyn FileSystem + Send))
     }
 }
 
 impl<'a> FileSystems<'a> for () {
-    type Iter = std::iter::Empty<&'a dyn FileSystem>;
+    type Iter = std::iter::Empty<&'a (dyn FileSystem + Send)>;
 
     fn filesystems(&'a self) -> Self::Iter {
         std::iter::empty()
@@ -84,14 +84,14 @@ macro_rules! tuple_filesystems {
             $first: FileSystem,
             $($rest: FileSystem),*
         {
-            type Iter = [&'a dyn FileSystem; count!($first $($rest)*)];
+            type Iter = [&'a (dyn FileSystem + Send); count!($first $($rest)*)];
 
             fn filesystems(&'a self) -> Self::Iter {
                 #[allow(non_snake_case)]
                 let ($first, $($rest),*) = self;
 
                 [
-                    $first as &dyn FileSystem,
+                    $first as &(dyn FileSystem + Send),
                     $($rest),*
                 ]
             }