feat(cache): let a listed folder lack entries a hit can rebuild

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Changelog
 
+- **Changed** When a task lists a folder, the files it wrote there and the folders holding only such files no longer count as entries of the listing: the next run is a cache hit if they are missing, as on a fresh clone, and the hit restores them. For example, `tsc` with no `include` lists the package root, which holds its `lib` output folder; removing `lib` used to be a cache miss. The folder's other entries still count ([#915](https://github.com/voidzero-dev/vite-task/pull/915)).
 - **Changed** A task that checks or lists a folder holding only files it wrote, like `tsc` checking its `dist` output folder, is now a cache hit when the folder is missing, as on a fresh clone, and the hit restores it. Previously the missing folder was a cache miss ([#914](https://github.com/voidzero-dev/vite-task/pull/914)).
 - **Changed** Tasks that write a file they also read are now cached, instead of being reported as `not cached because it modified its inputs`. For example, `tsc --incremental` reads and rewrites `tsconfig.tsbuildinfo`: the next run is a cache hit if the file is as `tsc` left it, or if it is missing, as on a fresh clone, in which case the hit restores it ([#913](https://github.com/voidzero-dev/vite-task/pull/913)).
 - **Fixed** A cache hit restores the symbolic links a task's `output` globs match as links, instead of dropping them, so a replayed output that links a directory, such as a bundler's `node_modules/<package>` link, is complete ([#885](https://github.com/voidzero-dev/vite-task/pull/885)).

crates/vt/docs/task-cache.md

@@ -197,7 +197,8 @@ Vite Task uses `fspy` to monitor file system access during task execution:
 │    • Check if file exists                                    │
 │    • If file: Hash contents with xxHash3                     │
 │    • If directory: Record structure                          │
-│      (skipped if it holds only files the task wrote)         │
+│      (skipped if it holds only files the task wrote; a       │
+│      listing leaves out such files and folders)              │
 │    • If missing: Mark as NotFound                            │
 │         │                                                    │
 │         ▼                                                    │

crates/vt/src/session/cache/mod.rs

@@ -276,7 +276,7 @@ pub fn split_path(path: &str) -> (Option<&str>, &str) {
 /// its own cache warm across branch switches, and a cache from a different
 /// version is simply ignored (it lives in a directory this build never looks
 /// at) rather than aborting the run. Bumping the version starts a fresh cache.
-const CACHE_SCHEMA_VERSION: u32 = 20;
+const CACHE_SCHEMA_VERSION: u32 = 21;
 
 /// Name of the per-version subdirectory (e.g. `v14`) under the task-cache
 /// directory that holds the database and output archives for the current

crates/vt/src/session/execute/fingerprint.rs

@@ -4,7 +4,7 @@
 //! fingerprints of file system state after task execution.
 
 use std::{
-    collections::BTreeMap,
+    collections::{BTreeMap, BTreeSet},
     ffi::OsStr,
     fs::File,
     io::{self, BufRead},
@@ -87,6 +87,10 @@ pub struct InferredInput {
     /// The task wrote the path, so a later run may find it missing, as on a
     /// clean checkout. A path that is there must still match `fingerprint`.
     pub may_be_missing: bool,
+    /// Entries of a folder listing that the task can rebuild, left out of
+    /// `fingerprint`. A later run may find them missing or present, so they're
+    /// left out of the current listing before comparing.
+    pub ignored_entries: BTreeSet<Str>,
 }
 
 /// Fingerprint for a single path (file or directory)
@@ -120,7 +124,8 @@ impl PostRunFingerprint {
     /// already tracked by the prerun glob fingerprint. Folders holding only
     /// what the task can rebuild are skipped too: a hit restores the files in
     /// them. Every other path is fingerprinted as the task left it, including
-    /// a file the task rewrote.
+    /// a file the task rewrote, except that a folder listing leaves out the
+    /// entries the task can rebuild and records their names.
     ///
     /// # Arguments
     /// * `inferred_path_reads` - Map of paths that were read during execution (from fspy)
@@ -145,9 +150,21 @@ impl PostRunFingerprint {
             })
             .map(|(relative_path, path_read)| {
                 let full_path = Arc::<AbsolutePath>::from(base_dir.join(relative_path));
-                let fingerprint = fingerprint_path(&full_path, *path_read)?;
+                let mut fingerprint = fingerprint_path(&full_path, *path_read)?;
+                let mut ignored_entries = BTreeSet::new();
+                if let PathFingerprint::Folder(Some(entries)) = &mut fingerprint {
+                    entries.retain(|name, kind| {
+                        let is_dir = matches!(kind, DirEntryKind::Dir);
+                        let ignored = rebuildable.is_rebuildable_entry(relative_path, name, is_dir);
+                        if ignored {
+                            ignored_entries.insert(name.clone());
+                        }
+                        !ignored
+                    });
+                }
                 let may_be_missing = rebuildable.may_be_missing(relative_path);
-                Ok((relative_path.clone(), InferredInput { fingerprint, may_be_missing }))
+                let input = InferredInput { fingerprint, may_be_missing, ignored_entries };
+                Ok((relative_path.clone(), input))
             })
             .collect::<anyhow::Result<HashMap<_, _>>>()?;
 
@@ -171,15 +188,23 @@ impl PostRunFingerprint {
     ) -> anyhow::Result<Option<PostRunMismatch>> {
         let input_mismatch =
             self.inferred_inputs.par_iter().find_map_any(|(input_relative_path, input)| {
-                let InferredInput { fingerprint: path_fingerprint, may_be_missing } = input;
+                let InferredInput {
+                    fingerprint: path_fingerprint,
+                    may_be_missing,
+                    ignored_entries,
+                } = input;
                 let input_full_path = Arc::<AbsolutePath>::from(base_dir.join(input_relative_path));
                 let path_read = PathRead {
                     read_dir_entries: matches!(path_fingerprint, PathFingerprint::Folder(Some(_))),
                 };
-                let current_path_fingerprint = match fingerprint_path(&input_full_path, path_read) {
-                    Ok(ok) => ok,
-                    Err(err) => return Some(Err(err)),
-                };
+                let mut current_path_fingerprint =
+                    match fingerprint_path(&input_full_path, path_read) {
+                        Ok(ok) => ok,
+                        Err(err) => return Some(Err(err)),
+                    };
+                if let PathFingerprint::Folder(Some(entries)) = &mut current_path_fingerprint {
+                    entries.retain(|name, _| !ignored_entries.contains(name));
+                }
                 if path_fingerprint == &current_path_fingerprint
                     || (*may_be_missing && current_path_fingerprint == PathFingerprint::NotFound)
                 {
@@ -763,4 +788,49 @@ mod tests {
         assert!(rebuildable.is_rebuildable_folder(&rel("out")));
         assert!(!rebuildable.is_rebuildable_folder(&rel("out/tmp")));
     }
+
+    #[test]
+    fn listings_leave_out_entries_the_task_can_rebuild() {
+        let tmp = tempfile::TempDir::new().unwrap();
+        let workspace_root = AbsolutePathBuf::new(tmp.path().to_path_buf()).unwrap();
+        let rel = |path: &str| RelativePathBuf::new(path).unwrap();
+        for dir in ["pkg/gen", "pkg/mixed"] {
+            std::fs::create_dir_all(workspace_root.join(dir).as_path()).unwrap();
+        }
+        let written = ["pkg/gen/a.txt", "pkg/out.txt", "pkg/mixed/a.txt"];
+        for file in written.iter().copied().chain(["pkg/mixed/notes.txt"]) {
+            std::fs::write(workspace_root.join(file).as_path(), "x").unwrap();
+        }
+
+        let path_reads = HashMap::from([(rel("pkg"), PathRead { read_dir_entries: true })]);
+        let fingerprint = PostRunFingerprint::create(
+            &path_reads,
+            &Rebuildable::new(written.into_iter().map(rel).collect(), &workspace_root),
+            &workspace_root,
+            &BTreeMap::new(),
+            BTreeMap::new(),
+            BTreeMap::new(),
+        )
+        .unwrap();
+        let listing = &fingerprint.inferred_inputs[&rel("pkg")];
+        assert_eq!(listing.ignored_entries, BTreeSet::from(["gen".into(), "out.txt".into()]));
+        assert_eq!(
+            listing.fingerprint,
+            PathFingerprint::Folder(Some(BTreeMap::from([("mixed".into(), DirEntryKind::Dir)])))
+        );
+
+        let validate = || fingerprint.validate(&workspace_root, &FxHashMap::default()).unwrap();
+        assert!(validate().is_none());
+        std::fs::remove_dir_all(workspace_root.join("pkg/gen").as_path()).unwrap();
+        std::fs::remove_file(workspace_root.join("pkg/out.txt").as_path()).unwrap();
+        assert!(validate().is_none());
+
+        std::fs::write(workspace_root.join("pkg/notes.txt").as_path(), "x").unwrap();
+        match validate() {
+            Some(PostRunMismatch::Input { kind: InputChangeKind::Added, path }) => {
+                assert_eq!(path.as_str(), "pkg/notes.txt");
+            }
+            other => panic!("expected `pkg/notes.txt` added, got {other:?}"),
+        }
+    }
 }

crates/vt/src/session/execute/rebuildable.rs

@@ -12,7 +12,9 @@
 //! which recreates the folder, so it isn't fingerprinted at all. Such folders
 //! are found after the run by climbing from the written paths through their
 //! parent folders, listing each one in the real file system. A written path
-//! or folder that is gone by then plays no part.
+//! or folder that is gone by then plays no part. A listing of any other
+//! folder leaves out its entries that the task can rebuild, which a later run
+//! may find missing too.
 //!
 //! The written paths are fspy's records before output exclusions and
 //! tool-reported `ignoreOutput` paths are applied: those decide what is
@@ -48,6 +50,14 @@ impl Rebuildable {
     pub fn is_rebuildable_folder(&self, path: &RelativePath) -> bool {
         self.folders.contains(path)
     }
+
+    /// Whether the entry `name` of the folder `dir`, itself a folder if
+    /// `is_dir`, is something the task can rebuild, so a listing of `dir` may
+    /// lack it.
+    pub fn is_rebuildable_entry(&self, dir: &RelativePath, name: &str, is_dir: bool) -> bool {
+        RelativePathBuf::new(name)
+            .is_ok_and(|name| is_rebuildable(&dir.join(name), is_dir, &self.written, &self.folders))
+    }
 }
 
 /// Finds the folders holding only what the task can rebuild. Each folder above
@@ -133,13 +143,21 @@ fn holds_only_rebuildable(
         let Ok(name) = RelativePathBuf::new(name) else {
             return Some(false);
         };
-        let path = dir.join(name);
-        let rebuildable =
-            if file_type.is_dir() { folders.contains(&path) } else { written.contains(&path) };
-        if !rebuildable {
+        if !is_rebuildable(&dir.join(name), file_type.is_dir(), written, folders) {
             return Some(false);
         }
         is_empty = false;
     }
     Some(!is_empty)
 }
+
+/// Whether the entry at `path` is rebuildable: a folder in `folders`, or a
+/// file (or symlink) in `written`.
+fn is_rebuildable(
+    path: &RelativePath,
+    is_dir: bool,
+    written: &FxHashSet<RelativePathBuf>,
+    folders: &FxHashSet<RelativePathBuf>,
+) -> bool {
+    if is_dir { folders.contains(path) } else { written.contains(path) }
+}

crates/vt_bin/tests/e2e_snapshots/fixtures/list_folder_holding_output/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "list-folder-holding-output",
+  "private": true
+}

crates/vt_bin/tests/e2e_snapshots/fixtures/list_folder_holding_output/snapshots.toml

@@ -0,0 +1,34 @@
+[[e2e]]
+name = "list_folder_holding_output"
+comment = """
+The task makes `gen`, writes `gen/a.txt` and lists the package root, like a code generator listing the folder its output goes into.
+"""
+steps = [
+  [
+    "vt",
+    "run",
+    "build",
+  ],
+  { argv = [
+    "vtt",
+    "rm",
+    "-r",
+    "gen",
+  ], comment = "remove the output folder, as on a clean checkout" },
+  [
+    "vt",
+    "run",
+    "build",
+  ],
+  { argv = [
+    "vtt",
+    "write-file",
+    "notes.txt",
+    "x",
+  ], comment = "other entries of the listing still count" },
+  [
+    "vt",
+    "run",
+    "build",
+  ],
+]

crates/vt_bin/tests/e2e_snapshots/fixtures/list_folder_holding_output/snapshots/list_folder_holding_output.md

@@ -0,0 +1,38 @@
+# list_folder_holding_output
+
+The task makes `gen`, writes `gen/a.txt` and lists the package root, like a code generator listing the folder its output goes into.
+
+## `vt run build`
+
+```
+$ vtt fs mkdir gen write gen/a.txt hi list .
+```
+
+## `vtt rm -r gen`
+
+remove the output folder, as on a clean checkout
+
+```
+```
+
+## `vt run build`
+
+```
+$ vtt fs mkdir gen write gen/a.txt hi list . ◉ cache hit, replaying
+
+---
+vt run: cache hit.
+```
+
+## `vtt write-file notes.txt x`
+
+other entries of the listing still count
+
+```
+```
+
+## `vt run build`
+
+```
+$ vtt fs mkdir gen write gen/a.txt hi list . ○ cache miss: 'notes.txt' added in workspace root, executing
+```

crates/vt_bin/tests/e2e_snapshots/fixtures/list_folder_holding_output/vite-task.json

@@ -0,0 +1,8 @@
+{
+  "tasks": {
+    "build": {
+      "command": "vtt fs mkdir gen write gen/a.txt hi list .",
+      "cache": true
+    }
+  }
+}

crates/vt_bin/tests/e2e_snapshots/fixtures/typescript_incremental/snapshots.toml

@@ -4,9 +4,6 @@ comment = """
 TypeScript 7's `tsc --incremental` keeps what it learned in `tsconfig.tsbuildinfo`. A run without the file probes for it and writes it, a run after an edit reads and rewrites it, and a run with nothing changed only reads it.
 """
 ignore = true
-# On Windows, tsc also lists the package root, so removing `tsconfig.tsbuildinfo` misses there
-# until listings leave out the files a task wrote.
-cfg = 'unix'
 steps = [
   { argv = [
     "vt",

crates/vt_bin/tests/e2e_snapshots/fixtures/typescript_lib_out_dir/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "typescript-lib-out-dir",
+  "private": true
+}

crates/vt_bin/tests/e2e_snapshots/fixtures/typescript_lib_out_dir/snapshots.toml

@@ -0,0 +1,24 @@
+[[e2e]]
+name = "typescript_lib_out_dir"
+comment = """
+TypeScript 7's `tsc` compiles `src` into `lib`. With no `include`, it lists the package root to find the files to compile, and stats `lib`.
+"""
+ignore = true
+steps = [
+  [
+    "vt",
+    "run",
+    "build",
+  ],
+  { argv = [
+    "vtt",
+    "rm",
+    "-r",
+    "lib",
+  ], comment = "remove the output folder, as on a clean checkout" },
+  [
+    "vt",
+    "run",
+    "build",
+  ],
+]

crates/vt_bin/tests/e2e_snapshots/fixtures/typescript_lib_out_dir/snapshots/typescript_lib_out_dir.md

@@ -0,0 +1,25 @@
+# typescript_lib_out_dir
+
+TypeScript 7's `tsc` compiles `src` into `lib`. With no `include`, it lists the package root to find the files to compile, and stats `lib`.
+
+## `vt run build`
+
+```
+$ tsc
+```
+
+## `vtt rm -r lib`
+
+remove the output folder, as on a clean checkout
+
+```
+```
+
+## `vt run build`
+
+```
+$ tsc ◉ cache hit, replaying
+
+---
+vt run: cache hit.
+```

crates/vt_bin/tests/e2e_snapshots/fixtures/typescript_lib_out_dir/src/main.ts

@@ -0,0 +1 @@
+export const value: string = 'initial';

crates/vt_bin/tests/e2e_snapshots/fixtures/typescript_lib_out_dir/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "compilerOptions": {
+    "strict": true,
+    "outDir": "lib",
+    "rootDir": "src"
+  }
+}

crates/vt_bin/tests/e2e_snapshots/fixtures/typescript_lib_out_dir/vite-task.json

@@ -0,0 +1,8 @@
+{
+  "tasks": {
+    "build": {
+      "command": "tsc",
+      "cache": true
+    }
+  }
+}