feat: persistent cache (#8687)

* feat: persistent cache

* fix: ci

Cargo.toml

@@ -182,6 +182,7 @@ rspack_plugin_wasm                     = { version = "0.2.0", path = "crates/rsp
 rspack_plugin_web_worker_template      = { version = "0.2.0", path = "crates/rspack_plugin_web_worker_template" }
 rspack_plugin_worker                   = { version = "0.2.0", path = "crates/rspack_plugin_worker" }
 rspack_regex                           = { version = "0.2.0", path = "crates/rspack_regex" }
+rspack_storage                         = { version = "0.2.0", path = "crates/rspack_storage" }
 rspack_swc_plugin_import               = { version = "0.2.0", path = "crates/swc_plugin_import" }
 rspack_testing                         = { version = "0.2.0", path = "crates/rspack_testing" }
 rspack_tracing                         = { version = "0.2.0", path = "crates/rspack_tracing" }

crates/node_binding/binding.d.ts

@@ -1379,8 +1379,10 @@ export interface RawExperimentCacheOptionsCommon {
 
 export interface RawExperimentCacheOptionsPersistent {
   type: "persistent"
+  buildDependencies: Array<string>
+  version: string
   snapshot: RawExperimentSnapshotOptions
-  storage: Array<RawStorageOptions>
+  storage: RawStorageOptions
 }
 
 export interface RawExperiments {

crates/node_binding/src/lib.rs

@@ -6,7 +6,7 @@ extern crate napi_derive;
 extern crate rspack_allocator;
 
 use std::pin::Pin;
-use std::sync::Mutex;
+use std::sync::{Arc, Mutex};
 
 use compiler::{Compiler, CompilerState, CompilerStateGuard};
 use napi::bindgen_prelude::*;
@@ -71,9 +71,9 @@ impl Rspack {
     let loader_resolver_factory = (*resolver_factory_reference)
       .get_loader_resolver_factory(compiler_options.resolve_loader.clone());
 
-    let intermediate_filesystem: Option<Box<dyn IntermediateFileSystem>> =
+    let intermediate_filesystem: Option<Arc<dyn IntermediateFileSystem>> =
       if let Some(fs) = intermediate_filesystem {
-        Some(Box::new(NodeFileSystem::new(fs).map_err(|e| {
+        Some(Arc::new(NodeFileSystem::new(fs).map_err(|e| {
           Error::from_reason(format!("Failed to create intermediate filesystem: {e}",))
         })?))
       } else {
@@ -85,7 +85,7 @@ impl Rspack {
       compiler_options,
       plugins,
       rspack_binding_options::buildtime_plugins::buildtime_plugins(),
-      Some(Box::new(NodeFileSystem::new(output_filesystem).map_err(
+      Some(Arc::new(NodeFileSystem::new(output_filesystem).map_err(
         |e| Error::from_reason(format!("Failed to create writable filesystem: {e}",)),
       )?)),
       intermediate_filesystem,

crates/rspack_binding_options/src/options/raw_experiments/raw_cache/mod.rs

@@ -22,23 +22,21 @@ pub struct RawExperimentCacheOptionsCommon {
 pub struct RawExperimentCacheOptionsPersistent {
   #[napi(ts_type = r#""persistent""#)]
   pub r#type: String,
-  //  pub build_dependencies: Vec<String>,
-  //  pub version: String,
+  pub build_dependencies: Vec<String>,
+  pub version: String,
   pub snapshot: RawExperimentSnapshotOptions,
-  pub storage: Vec<RawStorageOptions>,
+  pub storage: RawStorageOptions,
 }
 
 pub fn normalize_raw_experiment_cache_options(
   options: RawExperimentCacheOptions,
 ) -> ExperimentCacheOptions {
   match options {
     Either::A(persistent_options) => ExperimentCacheOptions::Persistent(PersistentCacheOptions {
+      build_dependencies: persistent_options.build_dependencies,
+      version: persistent_options.version,
       snapshot: persistent_options.snapshot.into(),
-      storage: persistent_options
-        .storage
-        .into_iter()
-        .map(Into::into)
-        .collect(),
+      storage: persistent_options.storage.into(),
     }),
     Either::B(options) => match options.r#type.as_str() {
       "disable" => ExperimentCacheOptions::Disabled,

crates/rspack_binding_options/src/options/raw_experiments/raw_cache/raw_storage.rs

@@ -10,7 +10,12 @@ pub struct RawStorageOptions {
 }
 
 impl From<RawStorageOptions> for StorageOptions {
-  fn from(_value: RawStorageOptions) -> Self {
-    StorageOptions::FileSystem
+  fn from(value: RawStorageOptions) -> Self {
+    match value.r#type.as_str() {
+      "filesystem" => StorageOptions::FileSystem {
+        directory: value.directory.into(),
+      },
+      s => panic!("unsupported storage type {s}"),
+    }
   }
 }

crates/rspack_core/Cargo.toml

@@ -42,6 +42,7 @@ rspack_paths = { workspace = true }
 rspack_regex = { workspace = true }
 rspack_resolver = { workspace = true }
 rspack_sources = { workspace = true }
+rspack_storage = { workspace = true }
 rspack_util = { workspace = true }
 rustc-hash = { workspace = true }
 serde = { workspace = true }

crates/rspack_core/src/cache/disable.rs

@@ -7,8 +7,9 @@ use crate::make::MakeArtifact;
 #[derive(Debug)]
 pub struct DisableCache;
 
+#[async_trait::async_trait]
 impl Cache for DisableCache {
-  fn before_make(&self, make_artifact: &mut MakeArtifact) {
+  async fn before_make(&self, make_artifact: &mut MakeArtifact) {
     *make_artifact = Default::default();
   }
 }

crates/rspack_core/src/cache/mod.rs

@@ -4,9 +4,9 @@ pub mod persistent;
 
 use std::{fmt::Debug, sync::Arc};
 
-use rspack_fs::FileSystem;
+use rspack_fs::{FileSystem, IntermediateFileSystem};
 
-use self::{disable::DisableCache, memory::MemoryCache, persistent::PersistentCache};
+pub use self::{disable::DisableCache, memory::MemoryCache, persistent::PersistentCache};
 use crate::{make::MakeArtifact, Compilation, CompilerOptions, ExperimentCacheOptions};
 
 /// Cache trait
@@ -21,25 +21,38 @@ use crate::{make::MakeArtifact, Compilation, CompilerOptions, ExperimentCacheOpt
 /// * This API does not need to cooperate with the js side.
 ///
 /// We can consider change to Hook when we need to open the API to js side.
+#[async_trait::async_trait]
 pub trait Cache: Debug + Send + Sync {
-  fn before_compile(&self, _compilation: &mut Compilation) {}
+  async fn before_compile(&self, _compilation: &mut Compilation) {}
   fn after_compile(&self, _compilation: &Compilation) {}
 
-  fn before_make(&self, _make_artifact: &mut MakeArtifact) {}
+  async fn before_make(&self, _make_artifact: &mut MakeArtifact) {}
   fn after_make(&self, _make_artifact: &MakeArtifact) {}
 }
 
 pub fn new_cache(
+  compiler_path: &String,
   compiler_option: Arc<CompilerOptions>,
   input_filesystem: Arc<dyn FileSystem>,
+  intermediate_filesystem: Arc<dyn IntermediateFileSystem>,
 ) -> Arc<dyn Cache> {
   match &compiler_option.experiments.cache {
     ExperimentCacheOptions::Disabled => Arc::new(DisableCache),
     ExperimentCacheOptions::Memory => Arc::new(MemoryCache),
-    ExperimentCacheOptions::Persistent(option) => Arc::new(PersistentCache::new(
-      option,
-      input_filesystem,
-      compiler_option.clone(),
-    )),
+    ExperimentCacheOptions::Persistent(option) => {
+      match PersistentCache::new(
+        compiler_path,
+        option,
+        compiler_option.clone(),
+        input_filesystem,
+        intermediate_filesystem,
+      ) {
+        Ok(cache) => Arc::new(cache),
+        Err(e) => {
+          tracing::warn!("create persistent cache failed {e:?}");
+          Arc::new(MemoryCache)
+        }
+      }
+    }
   }
 }

crates/rspack_core/src/cache/persistent/mod.rs

@@ -2,25 +2,28 @@ mod cacheable_context;
 mod occasion;
 pub mod snapshot;
 pub mod storage;
+mod version;
 use std::sync::Arc;
 
 pub use cacheable_context::{CacheableContext, FromContext};
 use occasion::MakeOccasion;
-use rspack_fs::FileSystem;
+use rspack_fs::{FileSystem, IntermediateFileSystem, Result};
 use rspack_paths::ArcPath;
 use rustc_hash::FxHashSet as HashSet;
 
 use self::{
   snapshot::{Snapshot, SnapshotOptions},
-  storage::{MemoryStorage, Storage, StorageOptions},
+  storage::{create_storage, Storage, StorageOptions},
 };
 use super::Cache;
 use crate::{make::MakeArtifact, Compilation, CompilerOptions};
 
 #[derive(Debug, Clone)]
 pub struct PersistentCacheOptions {
+  pub build_dependencies: Vec<String>,
+  pub version: String,
   pub snapshot: SnapshotOptions,
-  pub storage: Vec<StorageOptions>,
+  pub storage: StorageOptions,
 }
 
 /// Persistent cache implementation
@@ -33,29 +36,39 @@ pub struct PersistentCache {
 
 impl PersistentCache {
   pub fn new(
+    compiler_path: &String,
     option: &PersistentCacheOptions,
-    input_filesystem: Arc<dyn FileSystem>,
     compiler_options: Arc<CompilerOptions>,
-  ) -> Self {
-    let storage = Arc::new(MemoryStorage::default());
+    input_filesystem: Arc<dyn FileSystem>,
+    intermediate_filesystem: Arc<dyn IntermediateFileSystem>,
+  ) -> Result<Self> {
+    let version = version::get_version(
+      compiler_options.context.as_ref(),
+      input_filesystem.clone(),
+      &option.build_dependencies,
+      vec![compiler_path, &option.version],
+    )?;
+    let storage = create_storage(option.storage.clone(), version, intermediate_filesystem);
     let context = Arc::new(CacheableContext {
       options: compiler_options,
       input_filesystem: input_filesystem.clone(),
     });
     let make_occasion = MakeOccasion::new(storage.clone(), context);
-    Self {
+    Ok(Self {
       snapshot: Snapshot::new(option.snapshot.clone(), input_filesystem, storage.clone()),
       storage,
       make_occasion,
-    }
+    })
   }
 }
 
+#[async_trait::async_trait]
 impl Cache for PersistentCache {
-  fn before_compile(&self, compilation: &mut Compilation) {
+  async fn before_compile(&self, compilation: &mut Compilation) {
     if compilation.modified_files.is_empty() && compilation.removed_files.is_empty() {
       // inject modified_files and removed_files
-      let (modified_paths, removed_paths) = self.snapshot.calc_modified_paths();
+      let (modified_paths, removed_paths) = self.snapshot.calc_modified_paths().await;
+      tracing::info!("cache::snapshot recovery {modified_paths:?} {removed_paths:?}",);
       compilation.modified_files = modified_paths;
       compilation.removed_files = removed_paths;
     }
@@ -92,13 +105,17 @@ impl Cache for PersistentCache {
       .snapshot
       .add(modified_paths.iter().map(|item| item.as_ref()));
 
-    self.storage.trigger_save();
+    // TODO listen for storage finish in build mode
+    let _ = self.storage.trigger_save();
   }
 
-  fn before_make(&self, make_artifact: &mut MakeArtifact) {
+  async fn before_make(&self, make_artifact: &mut MakeArtifact) {
     if !make_artifact.initialized {
-      if let Ok(artifact) = self.make_occasion.recovery() {
-        *make_artifact = artifact;
+      match self.make_occasion.recovery().await {
+        Ok(artifact) => *make_artifact = artifact,
+        Err(err) => {
+          tracing::warn!("recovery error with {err:?}")
+        }
       }
     }
   }

crates/rspack_core/src/cache/persistent/occasion/make/dependencies.rs

@@ -10,7 +10,7 @@ use rustc_hash::FxHashMap as HashMap;
 use super::Storage;
 use crate::FileCounter;
 
-const SCOPE: &str = "occasion::make::dependencies";
+const SCOPE: &str = "occasion_make_dependencies";
 
 /// Dependency type
 #[cacheable]
@@ -109,39 +109,46 @@ pub fn save_dependencies_info(
     })
 }
 
-pub fn recovery_dependencies_info(
+pub async fn recovery_dependencies_info(
   storage: &Arc<dyn Storage>,
 ) -> Result<(FileCounter, FileCounter, FileCounter, FileCounter), DeserializeError> {
   let file_dep = Mutex::new(HashMap::default());
   let context_dep = Mutex::new(HashMap::default());
   let missing_dep = Mutex::new(HashMap::default());
   let build_dep = Mutex::new(HashMap::default());
-  storage.load(SCOPE).into_par_iter().try_for_each(|(k, v)| {
-    let count = usize::from_ne_bytes(
-      v.try_into()
-        .map_err(|_| DeserializeError::MessageError("deserialize count failed"))?,
-    );
-    let Dependency { r#type, path } = from_bytes(&k, &())?;
-    match r#type {
-      DepType::File => file_dep
-        .lock()
-        .expect("should get file dep")
-        .insert(path, count),
-      DepType::Context => context_dep
-        .lock()
-        .expect("should get context dep")
-        .insert(path, count),
-      DepType::Missing => missing_dep
-        .lock()
-        .expect("should get missing dep")
-        .insert(path, count),
-      DepType::Build => build_dep
-        .lock()
-        .expect("should get build dep")
-        .insert(path, count),
-    };
-    Ok(())
-  })?;
+  storage
+    .load(SCOPE)
+    .await
+    .unwrap_or_default()
+    .into_par_iter()
+    .try_for_each(|(k, v)| {
+      let count = usize::from_ne_bytes(
+        v.as_ref()
+          .clone()
+          .try_into()
+          .map_err(|_| DeserializeError::MessageError("deserialize count failed"))?,
+      );
+      let Dependency { r#type, path } = from_bytes(&k, &())?;
+      match r#type {
+        DepType::File => file_dep
+          .lock()
+          .expect("should get file dep")
+          .insert(path, count),
+        DepType::Context => context_dep
+          .lock()
+          .expect("should get context dep")
+          .insert(path, count),
+        DepType::Missing => missing_dep
+          .lock()
+          .expect("should get missing dep")
+          .insert(path, count),
+        DepType::Build => build_dep
+          .lock()
+          .expect("should get build dep")
+          .insert(path, count),
+      };
+      Ok(())
+    })?;
 
   Ok((
     FileCounter::new(file_dep.into_inner().expect("into_inner should be success")),

crates/rspack_core/src/cache/persistent/occasion/make/meta.rs

@@ -9,7 +9,7 @@ use rustc_hash::FxHashSet as HashSet;
 use super::Storage;
 use crate::{BuildDependency, DEPENDENCY_ID};
 
-const SCOPE: &str = "occasion::make::meta";
+const SCOPE: &str = "occasion_make_meta";
 
 /// The value struct of current storage scope
 #[cacheable]
@@ -44,10 +44,10 @@ pub fn save_meta(
   Ok(())
 }
 
-pub fn recovery_meta(
+pub async fn recovery_meta(
   storage: &Arc<dyn Storage>,
 ) -> Result<(HashSet<BuildDependency>, IdentifierSet), DeserializeError> {
-  let Some((_, value)) = storage.load(SCOPE).pop() else {
+  let Some((_, value)) = storage.load(SCOPE).await.unwrap_or_default().pop() else {
     return Err(DeserializeError::MessageError("can not get meta data"));
   };
   let meta: Meta = from_bytes(&value, &())?;