feat(cache): let a hit rebuild empty folders the task made

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Changelog
 
+- **Fixed** A cache hit now restores the empty folders a task made, like the `.astro/collections` folder that `astro build` leaves empty. A folder holding only such folders and files the task wrote, like `.astro`, may now be missing, as on a fresh clone, and the next run is still a cache hit that restores it. Previously a hit left empty folders missing, and removing a folder holding one was a cache miss ([#917](https://github.com/voidzero-dev/vite-task/pull/917)).
 - **Fixed** A task's outputs now include the files it renames into place, like Vitest's `fsModuleCache` entries written under a temporary name, every file in a folder it renames into place, like Vite's dependency optimizer renaming `deps_temp_*` to `deps`, and the hard links it creates. A cache hit restores them, and the next run is a cache hit if they are missing, as on a fresh clone. Previously they weren't restored, and removing one that the task read back was a cache miss ([#916](https://github.com/voidzero-dev/vite-task/pull/916)).
 - **Changed** When a task lists a folder, the files it wrote there and the folders holding only such files no longer count as entries of the listing: the next run is a cache hit if they are missing, as on a fresh clone, and the hit restores them. For example, `tsc` with no `include` lists the package root, which holds its `lib` output folder; removing `lib` used to be a cache miss. The folder's other entries still count ([#915](https://github.com/voidzero-dev/vite-task/pull/915)).
 - **Changed** A task that checks or lists a folder holding only files it wrote, like `tsc` checking its `dist` output folder, is now a cache hit when the folder is missing, as on a fresh clone, and the hit restores it. Previously the missing folder was a cache miss ([#914](https://github.com/voidzero-dev/vite-task/pull/914)).

crates/fspy/src/unix/syscall_handler/mod.rs

@@ -105,6 +105,8 @@ impl_handler!(
     faccessat,
     faccessat2,
 
+    #[cfg(target_arch = "x86_64")] mkdir,
+    mkdirat,
     #[cfg(target_arch = "x86_64")] rename,
     renameat,
     renameat2,

crates/fspy/src/unix/syscall_handler/namespace.rs

@@ -1,7 +1,9 @@
-//! Syscalls that put an existing file or folder at a new path without opening
-//! it: rename and hard link. The new path of a rename is recorded as a rename
-//! target, which may be a whole folder, and that of a hard link, always a
-//! file, as a write. The source of a rename needs no record.
+//! Syscalls that make an entry at a path without opening it: mkdir, and
+//! rename and hard link, which put an existing file or folder at a new path.
+//! The path of a mkdir is recorded as a mkdir, whether or not the folder is
+//! there already; the new path of a rename as a rename target, which may be
+//! a whole folder; and that of a hard link, always a file, as a write. The
+//! source of a rename needs no record.
 
 use std::io;
 
@@ -11,6 +13,19 @@ use fspy_shared::ipc::AccessMode;
 use super::SyscallHandler;
 
 impl SyscallHandler {
+    #[cfg(target_arch = "x86_64")]
+    pub(super) fn mkdir(&mut self, caller: Caller, (path,): (CStrPtr,)) -> io::Result<()> {
+        self.handle_path(caller, Fd::cwd(), path, AccessMode::Mkdir)
+    }
+
+    pub(super) fn mkdirat(
+        &mut self,
+        caller: Caller,
+        (dir_fd, path): (Fd, CStrPtr),
+    ) -> io::Result<()> {
+        self.handle_path(caller, dir_fd, path, AccessMode::Mkdir)
+    }
+
     #[cfg(target_arch = "x86_64")]
     pub(super) fn rename(
         &mut self,

crates/fspy/tests/rust_std.rs

@@ -69,6 +69,24 @@ async fn hard_link_destination() -> anyhow::Result<()> {
     Ok(())
 }
 
+#[test(tokio::test)]
+async fn mkdir() -> anyhow::Result<()> {
+    let tmp_dir = tempfile::tempdir()?;
+    fs::create_dir(tmp_dir.path().join("existing"))?;
+    let tmp_dir_str = tmp_dir.path().to_str().unwrap().to_owned();
+    let accesses = track_fn!(tmp_dir_str, |tmp_dir_str: String| {
+        let tmp_dir = std::path::Path::new(&tmp_dir_str);
+        fs::create_dir(tmp_dir.join("new")).unwrap();
+        // Fails, since the folder is there already; recorded anyway.
+        fs::create_dir(tmp_dir.join("existing")).unwrap_err();
+    })
+    .await?;
+    assert_contains(&accesses, tmp_dir.path().join("new").as_path(), PathAccess::Mkdir);
+    assert_contains(&accesses, tmp_dir.path().join("existing").as_path(), PathAccess::Mkdir);
+
+    Ok(())
+}
+
 #[test(tokio::test)]
 async fn metadata() -> anyhow::Result<()> {
     let tmp_dir = tempfile::tempdir()?;

crates/fspy/tests/static_executable.rs

@@ -89,6 +89,14 @@ async fn openat2_readwrite() {
     assert_contains(&accesses, Path::new("/hello"), fspy::PathAccess::ReadWrite);
 }
 
+#[test(tokio::test)]
+async fn mkdir() {
+    // The mkdir fails, since `/hello` doesn't exist; the folder is recorded
+    // anyway.
+    let accesses = track_test_bin(&["mkdir", "/hello/world"], None).await;
+    assert_contains(&accesses, Path::new("/hello/world"), fspy::PathAccess::Mkdir);
+}
+
 #[test(tokio::test)]
 async fn rename_destination() {
     // The rename fails, since `/hello` doesn't exist; the destination is

crates/fspy_e2e/src/main.rs

@@ -52,6 +52,7 @@ impl AccessCollector {
             PathAccess::RenameTarget(path) => {
                 relative(path).map(|path| format!("{path}: RenameTarget"))
             }
+            PathAccess::Mkdir(path) => relative(path).map(|path| format!("{path}: Mkdir")),
         };
         self.accesses.extend(line);
     }

crates/fspy_preload_unix/src/interceptions/namespace.rs

@@ -1,17 +1,34 @@
-//! Calls that put an existing file or folder at a new path without opening
-//! it: rename and hard link. The new path is recorded before the call, so a
-//! failed call is recorded too: as a rename target for a rename, which may
-//! move a whole folder, and as a write for a hard link, always to a file. The
-//! source of a rename needs no record.
+//! Calls that make an entry at a path without opening it: mkdir, and rename
+//! and hard link, which put an existing file or folder at a new path. The
+//! path is recorded before the call, so a failed call is recorded too: as a
+//! mkdir for a mkdir, whose folder may be there already; as a rename target
+//! for a rename, which may move a whole folder; and as a write for a hard
+//! link, always to a file. The source of a rename needs no record.
 
 use fspy_shared::ipc::AccessMode;
-use libc::{c_char, c_int};
+use libc::{c_char, c_int, mode_t};
 
 use crate::{
     client::{convert::PathAt, handle_open},
     macros::intercept,
 };
 
+intercept!(mkdir: unsafe extern "C" fn(path: *const c_char, mode: mode_t) -> c_int);
+unsafe extern "C" fn mkdir(path: *const c_char, mode: mode_t) -> c_int {
+    // SAFETY: path is a valid C string pointer provided by the caller of the interposed function
+    unsafe { handle_open(fspy_nostd::CStr::from_ptr(path.cast()), AccessMode::Mkdir) };
+    // SAFETY: calling the original libc mkdir() with the same arguments forwarded from the interposed function
+    unsafe { mkdir::original()(path, mode) }
+}
+
+intercept!(mkdirat: unsafe extern "C" fn(dirfd: c_int, path: *const c_char, mode: mode_t) -> c_int);
+unsafe extern "C" fn mkdirat(dirfd: c_int, path: *const c_char, mode: mode_t) -> c_int {
+    // SAFETY: dirfd is -1 or a valid file descriptor, and path is a valid C string, provided by the caller of the interposed function
+    unsafe { handle_open(PathAt::borrow_raw(dirfd, path), AccessMode::Mkdir) };
+    // SAFETY: calling the original libc mkdirat() with the same arguments forwarded from the interposed function
+    unsafe { mkdirat::original()(dirfd, path, mode) }
+}
+
 intercept!(rename: unsafe extern "C" fn(old: *const c_char, new: *const c_char) -> c_int);
 unsafe extern "C" fn rename(old: *const c_char, new: *const c_char) -> c_int {
     // SAFETY: new is a valid C string pointer provided by the caller of the interposed function

crates/fspy_preload_windows/src/windows/detours/nt.rs

@@ -3,10 +3,11 @@ use std::mem::{offset_of, size_of, size_of_val};
 use fspy_shared::ipc::{AccessMode, IpcPath, PathAccess};
 use ntapi::{
     ntioapi::{
-        FILE_INFORMATION_CLASS, FILE_RENAME_INFORMATION, FileLinkInformation,
-        FileLinkInformationEx, FileRenameInformation, FileRenameInformationEx,
-        NtQueryDirectoryFile, NtQueryFullAttributesFile, NtQueryInformationByName,
-        PFILE_BASIC_INFORMATION, PFILE_NETWORK_OPEN_INFORMATION, PIO_APC_ROUTINE, PIO_STATUS_BLOCK,
+        FILE_CREATE, FILE_DIRECTORY_FILE, FILE_INFORMATION_CLASS, FILE_OPEN_IF,
+        FILE_RENAME_INFORMATION, FileLinkInformation, FileLinkInformationEx, FileRenameInformation,
+        FileRenameInformationEx, NtQueryDirectoryFile, NtQueryFullAttributesFile,
+        NtQueryInformationByName, PFILE_BASIC_INFORMATION, PFILE_NETWORK_OPEN_INFORMATION,
+        PIO_APC_ROUTINE, PIO_STATUS_BLOCK,
     },
     ntpsapi::{
         NtCreateUserProcess, PPS_ATTRIBUTE_LIST, PPS_CREATE_INFO, PS_ATTRIBUTE,
@@ -160,6 +161,11 @@ unsafe fn read_process_image_attribute<'a>(
     None
 }
 
+// An open that may create a directory, from `CreateDirectoryW` (`FILE_CREATE`)
+// or a caller making it if it's missing (`FILE_OPEN_IF`), is recorded as a
+// mkdir, like `mkdir` on Unix, rather than as the read its
+// `FILE_LIST_DIRECTORY` access maps to. Listing the directory is recorded by
+// `NtQueryDirectoryFile`.
 static DETOUR_NT_CREATE_FILE: Detour<
     unsafe extern "system" fn(
         file_handle: PHANDLE,
@@ -191,8 +197,15 @@ static DETOUR_NT_CREATE_FILE: Detour<
                 ea_buffer: PVOID,
                 ea_length: ULONG,
             ) -> HFILE {
-                // SAFETY: intercepting file open to record access before forwarding to real function
-                unsafe { handle_open(desired_access, object_attributes) };
+                if create_options & FILE_DIRECTORY_FILE != 0
+                    && matches!(create_disposition, FILE_CREATE | FILE_OPEN_IF)
+                {
+                    // SAFETY: intercepting directory creation to record it before forwarding to real function
+                    unsafe { handle_open(AccessMode::Mkdir, object_attributes) };
+                } else {
+                    // SAFETY: intercepting file open to record access before forwarding to real function
+                    unsafe { handle_open(desired_access, object_attributes) };
+                }
 
                 // SAFETY: calling the original NtCreateFile with all original arguments
                 unsafe {

crates/fspy_shared/src/ipc/mod.rs

@@ -21,6 +21,7 @@ pub enum AccessMode {
     ReadWrite,
     ReadDir,
     RenameTarget,
+    Mkdir,
 }
 
 impl AccessMode {
@@ -32,6 +33,7 @@ impl AccessMode {
             Self::ReadWrite => PathAccess::ReadWrite(path),
             Self::ReadDir => PathAccess::ReadDir(path),
             Self::RenameTarget => PathAccess::RenameTarget(path),
+            Self::Mkdir => PathAccess::Mkdir(path),
         }
     }
 }
@@ -48,6 +50,9 @@ pub enum PathAccess<'a> {
     /// The new path of a rename. What is there after the call, a file or a
     /// whole folder, was put there without being opened.
     RenameTarget(&'a IpcPath),
+    /// A folder made by `mkdir`, recorded whether or not the call succeeded,
+    /// so the folder may have been there already.
+    Mkdir(&'a IpcPath),
 }
 
 impl<'a> PathAccess<'a> {
@@ -58,7 +63,8 @@ impl<'a> PathAccess<'a> {
             | PathAccess::Write(path)
             | PathAccess::ReadWrite(path)
             | PathAccess::ReadDir(path)
-            | PathAccess::RenameTarget(path) => iter::once(path),
+            | PathAccess::RenameTarget(path)
+            | PathAccess::Mkdir(path) => iter::once(path),
         }
     }
 
@@ -70,6 +76,7 @@ impl<'a> PathAccess<'a> {
             PathAccess::ReadWrite(path) => PathAccess::ReadWrite(path.clone_in(bump)),
             PathAccess::ReadDir(path) => PathAccess::ReadDir(path.clone_in(bump)),
             PathAccess::RenameTarget(path) => PathAccess::RenameTarget(path.clone_in(bump)),
+            PathAccess::Mkdir(path) => PathAccess::Mkdir(path.clone_in(bump)),
         }
     }
 }

crates/fspy_test_bin/src/main.rs

@@ -48,6 +48,9 @@ fn main() {
         "execve" => {
             let _ = std::process::Command::new(path).spawn();
         }
+        "mkdir" => {
+            let _ = std::fs::create_dir(path);
+        }
         "rename" => {
             let _ = std::fs::rename(path, second_path());
         }

crates/vt/docs/task-cache.md

@@ -197,7 +197,8 @@ Vite Task uses `fspy` to monitor file system access during task execution:
 │    • Check if file exists                                    │
 │    • If file: Hash contents with xxHash3                     │
 │    • If directory: Record structure                          │
-│      (skipped if it holds only files the task wrote; a       │
+│      (skipped if it's an empty folder the task made, or      │
+│      holds only files the task wrote and such folders; a     │
 │      listing leaves out such files and folders)              │
 │    • If missing: Mark as NotFound                            │
 │         │                                                    │

crates/vt/src/session/cache/archive.rs

@@ -7,26 +7,43 @@ use std::{
 
 use vt_path::{AbsolutePath, RelativePathBuf};
 
-/// Create a tar.zst archive from workspace-relative output paths.
+/// Create a tar.zst archive from workspace-relative output paths and output
+/// folder paths.
 ///
 /// A regular file is stored with its contents. A symbolic link is stored as a
 /// link with its target, never followed, so a link to a directory or a
-/// dangling link round-trips as the task left it. Paths that no longer exist
-/// are silently skipped (the task may delete temporary files during
-/// execution).
+/// dangling link round-trips as the task left it. A folder is stored without
+/// its contents, so that extracting recreates it even when nothing is in it.
+/// Paths that no longer exist are silently skipped (the task may delete
+/// temporary files during execution).
 ///
 /// # Errors
 ///
 /// Returns an error if creating the archive file or adding entries fails.
 pub fn create_output_archive(
     workspace_root: &AbsolutePath,
     output_files: &[RelativePathBuf],
+    output_folders: &[RelativePathBuf],
     archive_path: &AbsolutePath,
 ) -> anyhow::Result<()> {
     let file = File::create(archive_path.as_path())?;
     let encoder = zstd::Encoder::new(file, 0)?.auto_finish();
     let mut builder = tar::Builder::new(encoder);
 
+    for rel_path in output_folders {
+        let metadata = match std::fs::symlink_metadata(workspace_root.join(rel_path).as_path()) {
+            Ok(m) => m,
+            Err(err) if err.kind() == io::ErrorKind::NotFound => continue,
+            Err(err) => return Err(err.into()),
+        };
+        if metadata.is_dir() {
+            let mut header = tar::Header::new_gnu();
+            header.set_metadata(&metadata);
+            header.set_cksum();
+            builder.append_data(&mut header, rel_path.as_str(), io::empty())?;
+        }
+    }
+
     for rel_path in output_files {
         let abs_path = workspace_root.join(rel_path);
         // Skip files that no longer exist (task may delete temp files between
@@ -56,8 +73,8 @@ pub fn create_output_archive(
     Ok(())
 }
 
-/// Extract a tar.zst archive, restoring files and symbolic links relative to
-/// workspace root.
+/// Extract a tar.zst archive, restoring files, folders and symbolic links
+/// relative to workspace root.
 ///
 /// Parent directories are created automatically. Existing files are overwritten.
 ///
@@ -144,7 +161,7 @@ mod tests {
         let output = RelativePathBuf::new("output.txt").unwrap();
         std::fs::write(dir.join(&output).as_path(), "built\n".repeat(1000)).unwrap();
         let archive_path = dir.join("output.tar.zst");
-        create_output_archive(&dir, &[output], &archive_path).unwrap();
+        create_output_archive(&dir, &[output], &[], &archive_path).unwrap();
         let archive = std::fs::read(archive_path.as_path()).unwrap();
         check_output_archive(archive.as_slice()).unwrap();
 
@@ -187,7 +204,7 @@ mod tests {
         let files = ["dist/pkg/index.js", "dist/file.js", "dist/dependency", "dist/dangling"]
             .map(|path| RelativePathBuf::new(path).unwrap());
         let archive_path = dir.join("output.tar.zst");
-        create_output_archive(&dir, &files, &archive_path).unwrap();
+        create_output_archive(&dir, &files, &[], &archive_path).unwrap();
 
         fs::remove_dir_all(dir.join("dist")).unwrap();
         extract_output_archive(&dir, &archive_path).unwrap();
@@ -216,12 +233,30 @@ mod tests {
         symlink(outside.path().to_str().unwrap(), &dir.join("link"), true);
         let files = ["link", "link/sentinel.txt"].map(|path| RelativePathBuf::new(path).unwrap());
         let archive_path = dir.join("output.tar.zst");
-        create_output_archive(&dir, &files, &archive_path).unwrap();
+        create_output_archive(&dir, &files, &[], &archive_path).unwrap();
         fs::write(&sentinel, "must remain unchanged").unwrap();
 
         let restored = dir.join("restored");
         fs::create_dir_all(&restored).unwrap();
         assert!(extract_output_archive(&restored, &archive_path).is_err());
         assert_eq!(fs::read_to_string(&sentinel).unwrap(), "must remain unchanged");
     }
+
+    #[test]
+    fn extracting_recreates_empty_folders() {
+        let tmp = tempfile::TempDir::new().unwrap();
+        let source = AbsolutePathBuf::new(tmp.path().join("source")).unwrap();
+        let rel = |path: &str| RelativePathBuf::new(path).unwrap();
+        std::fs::create_dir_all(source.join("out/empty").as_path()).unwrap();
+        std::fs::write(source.join("out/a.txt").as_path(), "a").unwrap();
+        let archive_path = AbsolutePathBuf::new(tmp.path().join("output.tar.zst")).unwrap();
+        create_output_archive(&source, &[rel("out/a.txt")], &[rel("out/empty")], &archive_path)
+            .unwrap();
+
+        let target = AbsolutePathBuf::new(tmp.path().join("target")).unwrap();
+        std::fs::create_dir_all(target.as_path()).unwrap();
+        extract_output_archive(&target, &archive_path).unwrap();
+        assert!(target.join("out/empty").as_path().is_dir());
+        assert_eq!(std::fs::read_to_string(target.join("out/a.txt").as_path()).unwrap(), "a");
+    }
 }