feat: improve cache storage (#8675)

crates/rspack_fs/src/memory_fs.rs

@@ -310,7 +310,9 @@ impl ReadStream for MemoryReadStream {
   async fn read_until(&mut self, byte: u8) -> Result<Vec<u8>> {
     let mut buf = vec![];
     self.0.read_until(byte, &mut buf).map_err(Error::from)?;
-    buf.pop();
+    if buf.last().is_some_and(|b| b == &byte) {
+      buf.pop();
+    }
     Ok(buf)
   }
   async fn read_to_end(&mut self) -> Result<Vec<u8>> {

crates/rspack_fs/src/native_fs.rs

@@ -122,7 +122,9 @@ impl ReadStream for NativeReadStream {
   async fn read_until(&mut self, byte: u8) -> Result<Vec<u8>> {
     let mut buf = vec![];
     self.0.read_until(byte, &mut buf).map_err(Error::from)?;
-    buf.pop();
+    if buf.last().is_some_and(|b| b == &byte) {
+      buf.pop();
+    }
     Ok(buf)
   }
   async fn read_to_end(&mut self) -> Result<Vec<u8>> {

crates/rspack_storage/src/pack/data/meta.rs

@@ -1,6 +1,7 @@
 use std::time::{SystemTime, UNIX_EPOCH};
 
 use rspack_paths::{Utf8Path, Utf8PathBuf};
+use rustc_hash::FxHashSet as HashSet;
 
 use super::options::PackOptions;
 
@@ -12,14 +13,39 @@ pub struct PackFileMeta {
   pub wrote: bool,
 }
 
+#[derive(Debug, Default, Clone)]
+pub struct RootMeta {
+  pub last_modified: u64,
+  pub scopes: HashSet<String>,
+}
+
+impl RootMeta {
+  pub fn new(scopes: HashSet<String>) -> Self {
+    Self {
+      scopes,
+      last_modified: current_time(),
+    }
+  }
+  pub fn get_path(dir: &Utf8Path) -> Utf8PathBuf {
+    dir.join("storage_meta")
+  }
+}
+
+pub fn current_time() -> u64 {
+  SystemTime::now()
+    .duration_since(UNIX_EPOCH)
+    .expect("should get current time")
+    .as_millis() as u64
+}
+
 #[derive(Debug, Default)]
 pub struct ScopeMeta {
   pub path: Utf8PathBuf,
   pub bucket_size: usize,
   pub pack_size: usize,
-  pub last_modified: u64,
   pub packs: Vec<Vec<PackFileMeta>>,
 }
+
 impl ScopeMeta {
   pub fn new(dir: &Utf8Path, options: &PackOptions) -> Self {
     let mut packs = vec![];
@@ -30,15 +56,11 @@ impl ScopeMeta {
       path: Self::get_path(dir),
       bucket_size: options.bucket_size,
       pack_size: options.pack_size,
-      last_modified: SystemTime::now()
-        .duration_since(UNIX_EPOCH)
-        .expect("should get current time")
-        .as_millis() as u64,
       packs,
     }
   }
 
   pub fn get_path(dir: &Utf8Path) -> Utf8PathBuf {
-    dir.join("cache_meta")
+    dir.join("scope_meta")
   }
 }

crates/rspack_storage/src/pack/data/mod.rs

@@ -3,7 +3,7 @@ mod options;
 mod pack;
 mod scope;
 
-pub use meta::{PackFileMeta, ScopeMeta};
-pub use options::PackOptions;
+pub use meta::{current_time, PackFileMeta, RootMeta, ScopeMeta};
+pub use options::{PackOptions, RootOptions};
 pub use pack::{Pack, PackContents, PackKeys};
-pub use scope::PackScope;
+pub use scope::{PackScope, RootMetaState};

crates/rspack_storage/src/pack/data/options.rs

@@ -1,18 +1,14 @@
-use std::time::{SystemTime, UNIX_EPOCH};
+use rspack_paths::Utf8PathBuf;
 
 #[derive(Debug)]
 pub struct PackOptions {
   pub bucket_size: usize,
   pub pack_size: usize,
-  pub expire: u64,
 }
 
-impl PackOptions {
-  pub fn is_expired(&self, last_modified: &u64) -> bool {
-    let current_time = SystemTime::now()
-      .duration_since(UNIX_EPOCH)
-      .expect("get current time failed")
-      .as_millis() as u64;
-    current_time - last_modified > self.expire
-  }
+#[derive(Debug)]
+pub struct RootOptions {
+  pub root: Utf8PathBuf,
+  pub expire: u64,
+  pub clean: bool,
 }

crates/rspack_storage/src/pack/data/scope.rs

@@ -4,9 +4,25 @@ use itertools::Itertools;
 use rspack_paths::Utf8PathBuf;
 use rustc_hash::FxHashSet as HashSet;
 
-use super::{Pack, PackOptions, ScopeMeta};
+use super::{Pack, PackOptions, RootMeta, ScopeMeta};
 use crate::StorageContent;
 
+#[derive(Debug, Default)]
+pub enum RootMetaState {
+  #[default]
+  Pending,
+  Value(Option<RootMeta>),
+}
+
+impl RootMetaState {
+  pub fn expect_value(&self) -> &Option<RootMeta> {
+    match self {
+      RootMetaState::Value(v) => v,
+      RootMetaState::Pending => panic!("should have scope meta"),
+    }
+  }
+}
+
 #[derive(Debug, Default)]
 pub enum ScopeMetaState {
   #[default]