diff --git a/Cargo.lock b/Cargo.lock index 2ca650c8cc69a..77c27c66eb097 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2867,6 +2867,15 @@ dependencies = [ "log 0.4.20", ] +[[package]] +name = "file-id" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6584280525fb2059cba3db2c04abf947a1a29a45ddae89f3870f8281704fafc9" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "filedescriptor" version = "0.8.2" @@ -3023,25 +3032,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" -[[package]] -name = "fsevent" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab7d1bd1bd33cc98b0889831b72da23c0aa4df9cec7e0702f46ecea04b35db6" -dependencies = [ - "bitflags 1.3.2", - "fsevent-sys 2.0.1", -] - -[[package]] -name = "fsevent-sys" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f41b048a94555da0f42f1d632e2e19510084fb8e303b0daa2816e733fb3644a0" -dependencies = [ - "libc", -] - [[package]] name = "fsevent-sys" version = "4.1.0" @@ -3910,17 +3900,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f2cb48b81b1dc9f39676bf99f5499babfec7cd8fe14307f7b3d747208fb5690" -[[package]] -name = "inotify" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4816c66d2c8ae673df83366c18341538f234a26d65a9ecea5c348b453ac1d02f" -dependencies = [ - "bitflags 1.3.2", - "inotify-sys", - "libc", -] - [[package]] name = "inotify" version = "0.9.6" @@ -4953,18 +4932,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "mio-extras" -version = "2.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52403fe290012ce777c4626790c8951324a2b9e3316b3143779c72b029742f19" -dependencies = [ - "lazycell", - "log 0.4.20", - "mio 0.6.23", - "slab", -] - [[package]] name = "miow" version = "0.2.2" @@ -5238,24 +5205,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "notify" -version = "4.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae03c8c853dba7bfd23e571ff0cff7bc9dceb40a4cd684cd1681824183f45257" -dependencies = [ - "bitflags 1.3.2", - "filetime", - "fsevent", - "fsevent-sys 2.0.1", - "inotify 0.7.1", - "libc", - "mio 0.6.23", - "mio-extras", - "walkdir", - "winapi 0.3.9", -] - [[package]] name = "notify" version = "5.1.0" @@ -5265,8 +5214,8 @@ dependencies = [ "bitflags 1.3.2", "crossbeam-channel", "filetime", - "fsevent-sys 4.1.0", - "inotify 0.9.6", + "fsevent-sys", + "inotify", "kqueue", "libc", "mio 0.8.8", @@ -5283,8 +5232,8 @@ dependencies = [ "bitflags 2.4.0", "crossbeam-channel", "filetime", - "fsevent-sys 4.1.0", - "inotify 0.9.6", + "fsevent-sys", + "inotify", "kqueue", "libc", "log 0.4.20", @@ -5293,6 +5242,20 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "notify-debouncer-full" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f5dab59c348b9b50cf7f261960a20e389feb2713636399cd9082cd4b536154" +dependencies = [ + "crossbeam-channel", + "file-id", + "log 0.4.20", + "notify 6.1.1", + "parking_lot 0.12.1", + "walkdir", +] + [[package]] name = "notify-debouncer-mini" version = "0.2.1" @@ -10687,7 +10650,7 @@ dependencies = [ "indexmap 1.9.3", "jsonc-parser 0.21.0", "mime 0.3.17", - "notify 4.0.17", + "notify-debouncer-full", "parking_lot 0.12.1", "rstest", "serde", @@ -11645,7 +11608,7 @@ dependencies = [ "anyhow", "bitflags 1.3.2", "dashmap", - "fsevent-sys 4.1.0", + "fsevent-sys", "futures 0.3.28", "itertools 0.10.5", "libc", diff --git a/Cargo.toml b/Cargo.toml index c66fcb8ae589c..0a1679f17bb1c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -278,3 +278,4 @@ urlencoding = "2.1.2" webbrowser = "0.8.7" which = "4.4.0" unicode-segmentation = "1.10.1" +notify-debouncer-full = "0.3.1" diff --git a/crates/turbo-tasks-fs/Cargo.toml b/crates/turbo-tasks-fs/Cargo.toml index faa4bc2a73bfa..4e7dced8660c3 100644 --- a/crates/turbo-tasks-fs/Cargo.toml +++ b/crates/turbo-tasks-fs/Cargo.toml @@ -37,7 +37,7 @@ include_dir = { version = "0.7.2", features = ["nightly"] } indexmap = { workspace = true } jsonc-parser = { version = "0.21.0", features = ["serde"] } mime = { workspace = true } -notify = "4.0.17" +notify-debouncer-full = { workspace = true } parking_lot = { workspace = true } serde = { workspace = true, features = ["rc"] } serde_json = { workspace = true } diff --git a/crates/turbo-tasks-fs/benches/mod.rs b/crates/turbo-tasks-fs/benches/mod.rs index 7eb140df2b849..abb374f460a97 100644 --- a/crates/turbo-tasks-fs/benches/mod.rs +++ b/crates/turbo-tasks-fs/benches/mod.rs @@ -10,7 +10,10 @@ use criterion::{ measurement::{Measurement, WallTime}, BenchmarkId, Criterion, }; -use notify::{watcher, RecursiveMode, Watcher}; +use notify_debouncer_full::{ + new_debouncer, + notify::{RecursiveMode, Watcher}, +}; use tokio::runtime::Runtime; use turbo_tasks::event::Event; use turbo_tasks_fs::rope::{Rope, RopeBuilder}; @@ -35,8 +38,11 @@ fn bench_file_watching(c: &mut Criterion) { let (tx, rx) = channel(); let event = Arc::new(Event::new(|| "test event".to_string())); - let mut watcher = watcher(tx, Duration::from_micros(1)).unwrap(); - watcher.watch(temp_path, RecursiveMode::Recursive).unwrap(); + let mut watcher = new_debouncer(Duration::from_micros(1), None, tx).unwrap(); + watcher + .watcher() + .watch(temp_path, RecursiveMode::Recursive) + .unwrap(); let t = thread::spawn({ let event = event.clone(); diff --git a/crates/turbo-tasks-fs/src/lib.rs b/crates/turbo-tasks-fs/src/lib.rs index 96ee88df64871..bbb49f19faed7 100644 --- a/crates/turbo-tasks-fs/src/lib.rs +++ b/crates/turbo-tasks-fs/src/lib.rs @@ -48,7 +48,13 @@ use glob::Glob; use invalidator_map::InvalidatorMap; use jsonc_parser::{parse_to_serde_value, ParseOptions}; use mime::Mime; -use notify::{watcher, DebouncedEvent, RecommendedWatcher, RecursiveMode, Watcher}; +use notify_debouncer_full::{ + notify::{ + event::{MetadataKind, ModifyKind, RenameMode}, + EventKind, RecommendedWatcher, RecursiveMode, Watcher, + }, + DebouncedEvent, Debouncer, FileIdMap, +}; use read_glob::read_glob; pub use read_glob::ReadGlobResult; use serde::{Deserialize, Serialize}; @@ -104,7 +110,7 @@ pub trait FileSystem: ValueToString { #[derive(Default)] struct DiskWatcher { - watcher: Mutex>, + watcher: Mutex>>, /// Keeps track of which directories are currently watched. This is only /// used on a OS that doesn't support recursive watching. #[cfg(not(any(target_os = "macos", target_os = "windows")))] @@ -136,13 +142,13 @@ impl DiskWatcher { #[cfg(not(any(target_os = "macos", target_os = "windows")))] fn start_watching( &self, - watcher: &mut std::sync::MutexGuard>, + watcher: &mut std::sync::MutexGuard>>, dir_path: &Path, root_path: &Path, ) -> Result<()> { if let Some(watcher) = watcher.as_mut() { let mut path = dir_path; - while let Err(err) = watcher.watch(path, RecursiveMode::NonRecursive) { + while let Err(err) = watcher.watcher().watch(path, RecursiveMode::NonRecursive) { if path == root_path { return Err(err).context(format!( "Unable to watch {} (tried up to {})", @@ -264,6 +270,23 @@ impl DiskFileSystem { self.start_watching_internal(true) } + /// + /// Create a watcher and start watching by creating `debounced` watcher + /// via `full debouncer` + /// + /// `notify` provides 2 different debouncer implementation, `-full` + /// provides below differences for the easy of use: + /// + /// - Only emits a single Rename event if the rename From and To events can + /// be matched + /// - Merges multiple Rename events + /// - Takes Rename events into account and updates paths for events that + /// occurred before the rename event, but which haven't been emitted, yet + /// - Optionally keeps track of the file system IDs all files and stiches + /// rename events together (FSevents, Windows) + /// - Emits only one Remove event when deleting a directory (inotify) + /// - Doesn't emit duplicate create events + /// - Doesn't emit Modify events after a Create event fn start_watching_internal(&self, report_invalidation_reason: bool) -> Result<()> { let mut watcher_guard = self.watcher.watcher.lock().unwrap(); if watcher_guard.is_some() { @@ -288,14 +311,19 @@ impl DiskFileSystem { let delay = Duration::from_millis(1); // Create a watcher object, delivering debounced events. // The notification back-end is selected based on the platform. - let mut watcher = watcher(tx, delay)?; + let mut debounced_watcher = notify_debouncer_full::new_debouncer(delay, None, tx)?; // Add a path to be watched. All files and directories at that path and // below will be monitored for changes. #[cfg(any(target_os = "macos", target_os = "windows"))] - watcher.watch(&root_path, RecursiveMode::Recursive)?; + debounced_watcher + .watcher() + .watch(&root_path, RecursiveMode::Recursive)?; + #[cfg(not(any(target_os = "macos", target_os = "windows")))] for dir_path in self.watcher.watching.iter() { - watcher.watch(&*dir_path, RecursiveMode::NonRecursive)?; + debounced_watcher + .watcher() + .watch(&dir_path, RecursiveMode::NonRecursive)?; } // We need to invalidate all reads that happened before watching @@ -323,7 +351,7 @@ impl DiskFileSystem { }); } - watcher_guard.replace(watcher); + watcher_guard.replace(debounced_watcher); drop(watcher_guard); #[cfg(not(any(target_os = "macos", target_os = "windows")))] @@ -343,60 +371,103 @@ impl DiskFileSystem { }); loop { match event { - Ok(DebouncedEvent::Write(path)) => { - batched_invalidate_path.insert(path); - } - Ok(DebouncedEvent::Create(path)) => { - batched_invalidate_path_and_children.insert(path.clone()); - batched_invalidate_path_and_children_dir.insert(path.clone()); - if let Some(parent) = path.parent() { - batched_invalidate_path_dir.insert(PathBuf::from(parent)); - } - #[cfg(not(any(target_os = "macos", target_os = "windows")))] - batched_new_paths.insert(path.clone()); - } - Ok(DebouncedEvent::Remove(path)) => { - batched_invalidate_path_and_children.insert(path.clone()); - batched_invalidate_path_and_children_dir.insert(path.clone()); - if let Some(parent) = path.parent() { - batched_invalidate_path_dir.insert(PathBuf::from(parent)); - } - } - Ok(DebouncedEvent::Rename(source, destination)) => { - batched_invalidate_path_and_children.insert(source.clone()); - if let Some(parent) = source.parent() { - batched_invalidate_path_dir.insert(PathBuf::from(parent)); - } - batched_invalidate_path_and_children.insert(destination.clone()); - if let Some(parent) = destination.parent() { - batched_invalidate_path_dir.insert(PathBuf::from(parent)); - } - #[cfg(not(any(target_os = "macos", target_os = "windows")))] - batched_new_paths.insert(destination.clone()); - } - Ok(DebouncedEvent::Rescan) => { - batched_invalidate_path_and_children.insert(PathBuf::from(&root)); - batched_invalidate_path_and_children_dir.insert(PathBuf::from(&root)); - } - Ok(DebouncedEvent::Error(err, path)) => { - println!("watch error ({:?}): {:?} ", path, err); - match path { - Some(path) => { - batched_invalidate_path_and_children.insert(path.clone()); - batched_invalidate_path_and_children_dir.insert(path); + Ok(Ok(events)) => { + events.iter().for_each(|DebouncedEvent { event: notify_debouncer_full::notify::Event {kind, paths, ..}, .. }| { + // [NOTE] there is attrs in the `Event` struct, which contains few more metadata like process_id who triggered the event, + // or the source we may able to utilize later. + match kind { + // [NOTE] Observing `ModifyKind::Metadata(MetadataKind::Any)` is not a mistake, fix for PACK-2437. + // In here explicitly subscribes to the `ModifyKind::Data` which + // indicates file content changes - in case of fsevents backend, this is `kFSEventStreamEventFlagItemModified`. + // Also meanwhile we subscribe to ModifyKind::Metadata as well. + // This is due to in some cases fsevents does not emit explicit kFSEventStreamEventFlagItemModified kernel events, + // but only emits kFSEventStreamEventFlagItemInodeMetaMod. While this could cause redundant invalidation, + // it's the way to reliably detect file content changes. + // ref other implementation, i.e libuv does same thing to trigger UV_CHANEGS + // https://github.com/libuv/libuv/commit/73cf3600d75a5884b890a1a94048b8f3f9c66876#diff-e12fdb1f404f1c97bbdcc0956ac90d7db0d811d9fa9ca83a3deef90c937a486cR95-R99 + EventKind::Modify(ModifyKind::Data(_) | ModifyKind::Metadata(MetadataKind::Any)) => { + batched_invalidate_path.extend(paths.clone()); + } + EventKind::Create(_) => { + batched_invalidate_path_and_children.extend(paths.clone()); + batched_invalidate_path_and_children_dir.extend(paths.clone()); + paths.iter().for_each(|path| { + if let Some(parent) = path.parent() { + batched_invalidate_path_dir.insert(PathBuf::from(parent)); + } + }); + + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + batched_new_paths.extend(paths.clone()); + } + EventKind::Remove(_) => { + batched_invalidate_path_and_children.extend(paths.clone()); + batched_invalidate_path_and_children_dir.extend(paths.clone()); + paths.iter().for_each(|path| { + if let Some(parent) = path.parent() { + batched_invalidate_path_dir.insert(PathBuf::from(parent)); + } + }); + } + // A single event emitted with both the `From` and `To` paths. + EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => { + // For the rename::both, notify provides an array of paths in given order + if let [source, destination, ..] = &paths[..] { + batched_invalidate_path_and_children.insert(source.clone()); + if let Some(parent) = source.parent() { + batched_invalidate_path_dir.insert(PathBuf::from(parent)); + } + batched_invalidate_path_and_children.insert(destination.clone()); + if let Some(parent) = destination.parent() { + batched_invalidate_path_dir.insert(PathBuf::from(parent)); + } + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + batched_new_paths.insert(destination.clone()); + } else { + // If we hit here, we expect this as a bug either in notify or system weirdness. + panic!("Rename event does not contain source and destination paths {:#?}", paths); + } + } + // We expect RenameMode::Both covers most of the case we need to invalidate, + // but also checks RenameMode::To just in case to avoid edge cases. + EventKind::Any | EventKind::Modify(ModifyKind::Any | ModifyKind::Name(RenameMode::To)) => { + batched_invalidate_path.extend(paths.clone()); + batched_invalidate_path_and_children.extend(paths.clone()); + batched_invalidate_path_and_children_dir.extend(paths.clone()); + for parent in paths.iter().filter_map(|path| path.parent()) { + batched_invalidate_path_dir.insert(PathBuf::from(parent)); + } + } + EventKind::Modify( + ModifyKind::Metadata(..) + | ModifyKind::Other + | ModifyKind::Name(..) + ) + | EventKind::Access(_) + | EventKind::Other => { + // ignored + } } - None => { - batched_invalidate_path_and_children - .insert(PathBuf::from(&root)); - batched_invalidate_path_and_children_dir - .insert(PathBuf::from(&root)); - } - } + }); } - Ok(DebouncedEvent::Chmod(_)) - | Ok(DebouncedEvent::NoticeRemove(_)) - | Ok(DebouncedEvent::NoticeWrite(_)) => { - // ignored + // Error raised by notify watcher itself + Ok(Err(errors)) => { + errors.iter().for_each( + |notify_debouncer_full::notify::Error { kind, paths }| { + println!("watch error ({:?}): {:?} ", paths, kind); + + if paths.is_empty() { + batched_invalidate_path_and_children + .insert(PathBuf::from(&root)); + batched_invalidate_path_and_children_dir + .insert(PathBuf::from(&root)); + } else { + batched_invalidate_path_and_children.extend(paths.clone()); + batched_invalidate_path_and_children_dir + .extend(paths.clone()); + } + }, + ); } Err(TryRecvError::Disconnected) => { // Sender has been disconnected