feat(inputs): add doublestar to dir globs

crates/turborepo-globwalk/src/lib.rs

@@ -15,7 +15,8 @@ use path_clean::PathClean;
 use path_slash::PathExt;
 use rayon::prelude::*;
 use regex::Regex;
-use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, PathError};
+use tracing::debug;
+use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, PathError, RelativeUnixPath};
 use wax::{walk::FileIterator, BuildError, Glob};
 
 #[derive(Debug, PartialEq, Clone, Copy)]
@@ -51,10 +52,13 @@ fn join_unix_like_paths(a: &str, b: &str) -> String {
     [a.trim_end_matches('/'), "/", b.trim_start_matches('/')].concat()
 }
 
-fn escape_glob_literals(literal_glob: &str) -> Cow<str> {
+fn glob_literals() -> &'static Regex {
     static RE: OnceLock<Regex> = OnceLock::new();
     RE.get_or_init(|| Regex::new(r"(?<literal>[\?\*\$:<>\(\)\[\]{},])").unwrap())
-        .replace_all(literal_glob, "\\$literal")
+}
+
+fn escape_glob_literals(literal_glob: &str) -> Cow<str> {
+    glob_literals().replace_all(literal_glob, "\\$literal")
 }
 
 #[tracing::instrument]
@@ -63,6 +67,8 @@ fn preprocess_paths_and_globs(
     include: &[String],
     exclude: &[String],
 ) -> Result<(PathBuf, Vec<String>, Vec<String>), WalkError> {
+    debug!("processing includes: {include:?}");
+    debug!("processing excludes: {exclude:?}");
     let base_path_slash = base_path
         .as_std_path()
         .to_slash()
@@ -76,6 +82,12 @@ fn preprocess_paths_and_globs(
     let (include_paths, lowest_segment) = include
         .iter()
         .map(|s| fix_glob_pattern(s))
+        .map(|mut s| {
+            // We need to check inclusion globs before the join
+            // as to_slash doesn't preserve Windows drive names.
+            add_doublestar_to_dir(base_path, &mut s);
+            s
+        })
         .map(|s| join_unix_like_paths(&base_path_slash, &s))
         .filter_map(|s| collapse_path(&s).map(|(s, v)| (s.to_string(), v)))
         .fold(
@@ -106,6 +118,8 @@ fn preprocess_paths_and_globs(
         // unless it already ends with a double star
         add_trailing_double_star(&mut exclude_paths, &split);
     }
+    debug!("processed includes: {include_paths:?}");
+    debug!("processed excludes: {exclude_paths:?}");
 
     Ok((base_path, include_paths, exclude_paths))
 }
@@ -211,13 +225,46 @@ fn add_trailing_double_star(exclude_paths: &mut Vec<String>, glob: &str) {
         exclude_paths.push(glob.to_string());
     } else {
         // Match Go globby behavior. If the glob doesn't already end in /**, add it
-        // TODO: The Go version uses system separator. Are we forcing all globs to unix
-        // paths?
+        // We use the unix style operator as wax expects unix style paths
         exclude_paths.push(format!("{}/**", glob));
         exclude_paths.push(glob.to_string());
     }
 }
 
+fn add_doublestar_to_dir(base: &AbsoluteSystemPath, glob: &mut String) {
+    // If the glob has a glob literal in it e.g. *
+    // then skip trying to read it as a file path.
+    if glob_literals().is_match(&*glob) {
+        return;
+    }
+
+    // Globs are given in unix style
+    let Ok(glob_path) = RelativeUnixPath::new(&*glob) else {
+        // Glob isn't valid relative unix path so can't check if dir
+        debug!("'{glob}' isn't valid path");
+        return;
+    };
+
+    let path = base.join_unix_path(glob_path);
+
+    let Ok(metadata) = path.symlink_metadata() else {
+        debug!("'{path}' doesn't have metadata");
+        return;
+    };
+
+    if !metadata.is_dir() {
+        return;
+    }
+
+    debug!("'{path}' is a directory");
+
+    // Glob points to a dir, must add **
+    if !glob.ends_with('/') {
+        glob.push('/');
+    }
+    glob.push_str("**");
+}
+
 #[tracing::instrument]
 fn glob_with_contextual_error<S: AsRef<str> + std::fmt::Debug>(
     raw: S,
@@ -373,12 +420,13 @@ mod test {
     use std::{collections::HashSet, str::FromStr};
 
     use itertools::Itertools;
+    use tempdir::TempDir;
     use test_case::test_case;
-    use turbopath::AbsoluteSystemPathBuf;
+    use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf};
 
     use crate::{
-        collapse_path, escape_glob_literals, fix_glob_pattern, globwalk, ValidatedGlob, WalkError,
-        WalkType,
+        add_doublestar_to_dir, collapse_path, escape_glob_literals, fix_glob_pattern, globwalk,
+        ValidatedGlob, WalkError, WalkType,
     };
 
     #[cfg(unix)]
@@ -1426,4 +1474,26 @@ mod test {
             r"\?\*\$\:\<\>\(\)\[\]\{\}\,"
         );
     }
+
+    #[test_case("foo", false, "foo" ; "file")]
+    #[test_case("foo", true, "foo/**" ; "dir")]
+    #[test_case("foo/", true, "foo/**" ; "dir slash")]
+    #[test_case("f[o0]o", true, "f[o0]o" ; "non-literal")]
+    fn test_add_double_star(glob: &str, is_dir: bool, expected: &str) {
+        let tmpdir = TempDir::new("doublestar").unwrap();
+        let base = AbsoluteSystemPath::new(tmpdir.path().to_str().unwrap()).unwrap();
+
+        let foo = base.join_component("foo");
+
+        match is_dir {
+            true => foo.create_dir_all().unwrap(),
+            false => foo.create_with_contents(b"bar").unwrap(),
+        }
+
+        let mut glob = glob.to_owned();
+
+        add_doublestar_to_dir(base, &mut glob);
+
+        assert_eq!(glob, expected);
+    }
 }

turborepo-tests/integration/fixtures/dir_globs/.gitignore

@@ -0,0 +1,3 @@
+node_modules/
+.turbo
+.npmrc

turborepo-tests/integration/fixtures/dir_globs/apps/my-app/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "my-app",
+  "scripts": {
+    "build": "echo building",
+    "maybefails": "exit 4"
+  },
+  "dependencies": {
+    "util": "*"
+  }
+}

turborepo-tests/integration/fixtures/dir_globs/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "monorepo",
+  "scripts": {
+    "something": "turbo run build"
+  },
+  "workspaces": [
+    "apps/**",
+    "packages/**"
+  ]
+}

turborepo-tests/integration/fixtures/dir_globs/packages/util/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "util",
+  "scripts": {
+    "build": "echo building; mkdir dist; echo 'world' > dist/hello.txt "
+  }
+}

turborepo-tests/integration/fixtures/dir_globs/packages/util/src/boo.txt

@@ -0,0 +1 @@
+one

turborepo-tests/integration/fixtures/dir_globs/turbo.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "https://turbo.build/schema.json",
+  "pipeline": {
+    "build": {
+      "inputs": ["src/"],
+      "outputs": ["dist"]
+    }
+  }
+}

turborepo-tests/integration/tests/run/globs.t

@@ -0,0 +1,39 @@
+Setup
+  $ . ${TESTDIR}/../../../helpers/setup_integration_test.sh dir_globs
+Verify that input directory change causes cache miss
+  $ ${TURBO} build --filter=util --output-logs=hash-only
+  \xe2\x80\xa2 Packages in scope: util (esc)
+  \xe2\x80\xa2 Running build in 1 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  util:build: cache miss, executing 1b15024312fdecef
+
+   Tasks:    1 successful, 1 total
+  Cached:    0 cached, 1 total
+    Time:\s*[\.0-9]+m?s  (re)
+
+  $ touch packages/util/src/oops.txt
+  $ ${TURBO} build --filter=util --output-logs=hash-only
+  \xe2\x80\xa2 Packages in scope: util (esc)
+  \xe2\x80\xa2 Running build in 1 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  util:build: cache miss, executing 31adf3c5481a64c7
+
+   Tasks:    1 successful, 1 total
+  Cached:    0 cached, 1 total
+    Time:\s*[\.0-9]+m?s  (re)
+
+  $ cat packages/util/dist/hello.txt
+  world
+  $ rm packages/util/dist/hello.txt
+  $ ${TURBO} build --filter=util --output-logs=hash-only
+  \xe2\x80\xa2 Packages in scope: util (esc)
+  \xe2\x80\xa2 Running build in 1 packages (esc)
+  \xe2\x80\xa2 Remote caching disabled (esc)
+  util:build: cache hit, suppressing logs 31adf3c5481a64c7
+
+   Tasks:    1 successful, 1 total
+  Cached:    1 cached, 1 total
+    Time:\s*[\.0-9]+m?s >>> FULL TURBO (re)
+
+  $ cat packages/util/dist/hello.txt
+  world