Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 109 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -495,6 +495,115 @@ jobs:
cargo test --locked -p telex-watcher --no-default-features --features sqlite
--lib ownership::tests:: -- --nocapture

watcher-containment-macos:
# Native conformance for the macOS assigned-process-group guard (#172,
# ADR 0054 as amended by ADR 0057). Each run is evidence for the image it
# ran on only; the set of images here is coverage, never a runtime
# allowlist. Admission has no OS-version gate.
name: Watcher containment conformance (${{ matrix.os }})
runs-on: ${{ matrix.os }}
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
os:
- macos-15
- macos-26
- macos-15-intel
env:
RUSTFLAGS: -D warnings
steps:
- name: Checkout exact source head
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}

- name: Record the actual host this evidence covers
env:
EXPECTED_SOURCE_HEAD: ${{ github.event.pull_request.head.sha || github.sha }}
run: |
test "$(git rev-parse HEAD)" = "$EXPECTED_SOURCE_HEAD"
echo "source=$EXPECTED_SOURCE_HEAD runner_label=${{ matrix.os }} image=${ImageOS:-unknown}/${ImageVersion:-unknown}"
sw_vers
echo "process_arch=$(uname -m)"
sysctl kern.version kern.osversion
sysctl hw.optional.arm64 || echo "hw.optional.arm64: absent (Intel hardware)"
sysctl sysctl.proc_translated || echo "sysctl.proc_translated: absent (native Intel)"
umask 077
echo "TELEX_TEST_BASE_DIR=$(mktemp -d /private/tmp/XXXXXX)" >> "$GITHUB_ENV"
home_root="$(sudo mktemp -d /Users/XXXXXX)"
sudo chown "$(id -u):$(id -g)" "$home_root"
chmod 0700 "$home_root"
echo "TELEX_TEST_HOME_ROOT=$home_root" >> "$GITHUB_ENV"

- name: Install Rust stable
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: stable
target: ${{ runner.arch == 'ARM64' && 'x86_64-apple-darwin' || '' }}

- name: Build translated guard and fixture (Apple Silicon with Rosetta)
if: runner.arch == 'ARM64'
run: |
if arch -x86_64 /usr/bin/true 2>/dev/null; then
cargo build --locked -p telex-watcher --no-default-features --features sqlite \
--target x86_64-apple-darwin --bin watcher_process_guard --bin watcher_process_fixture
{
echo "WATCHER_TEST_X86_64_GUARD=$PWD/target/x86_64-apple-darwin/debug/watcher_process_guard"
echo "WATCHER_TEST_X86_64_FIXTURE=$PWD/target/x86_64-apple-darwin/debug/watcher_process_fixture"
} >> "$GITHUB_ENV"
else
echo "NOT COVERED on this image: Rosetta is unavailable, translated cases are skipped"
fi

- name: Admission, census and signal-authority unit checks
run: >-
cargo test --locked -p telex-watcher --no-default-features --features sqlite
--lib process:: -- --nocapture

- name: Process containment and native conformance
run: >-
cargo test --locked -p telex-watcher --no-default-features --features sqlite
--test process_v2 -- --nocapture

- name: Bounded PID/PGID reuse after Ended
run: >-
cargo test --locked -p telex-watcher --no-default-features --features sqlite
--test process_v2 -- --nocapture --ignored
macos_conformance::reused_pid_and_pgid_are_never_signalled_after_ended

- name: Start isolated PostgreSQL for native sender parity
run: |
brew list postgresql@17 >/dev/null 2>&1 || brew install postgresql@17
pg_bin="$(brew --prefix postgresql@17)/bin"
pg_data="$RUNNER_TEMP/watcher-postgres"
"$pg_bin/initdb" -D "$pg_data" -U postgres --auth=trust --no-locale
"$pg_bin/pg_ctl" -D "$pg_data" -l "$RUNNER_TEMP/watcher-postgres.log" \
-o "-h 127.0.0.1 -p 55472 -k $TELEX_TEST_BASE_DIR" -w start
echo "WATCHER_PG_BIN=$pg_bin" >> "$GITHUB_ENV"
echo "WATCHER_PG_DATA=$pg_data" >> "$GITHUB_ENV"
echo "TELEX_PG_URL=host=127.0.0.1 port=55472 user=postgres dbname=postgres sslmode=disable" >> "$GITHUB_ENV"
echo "TELEX_PG_REQUIRE=1" >> "$GITHUB_ENV"
echo "TELEX_PG_PASSWORD=watcher-local-test-only" >> "$GITHUB_ENV"

- name: Build matching native daemon, Watcher and test guardians
run: |
cargo build --locked -p telex -p telex-watcher --no-default-features \
--features sqlite,postgres --bins
set -o pipefail
cargo test --locked -p telex-watcher --no-default-features --features sqlite,postgres \
--no-run --message-format=json | python3 tests/stage_watcher_test_guard.py

- name: Prove native SQLite and PostgreSQL lifecycle, sender readiness and health
run: |
cargo test --locked -p telex-watcher --no-default-features --features sqlite,postgres \
--lib --test installation --test runtime_diagnostics --test cli_v2 \
-- --test-threads=1 --nocapture

- name: Stop isolated PostgreSQL
if: always() && env.WATCHER_PG_DATA != ''
run: '"$WATCHER_PG_BIN/pg_ctl" -D "$WATCHER_PG_DATA" -m immediate -w stop'

daemon-bootstrap-identity-macos:
name: Daemon bootstrap image identity (macOS)
runs-on: macos-latest
Expand Down
19 changes: 10 additions & 9 deletions docs/design/watcher.md
Original file line number Diff line number Diff line change
Expand Up @@ -726,13 +726,12 @@ Watcher applies an explicit host-specific containment guarantee:
subreaper guardian, including descendants that change groups or sessions.
The guardian's startup handshake and its terminal-receipt path/nonce identity
are host-required before detector release.
- The accepted macOS contract is for conforming non-detaching detectors. When
the capability is enabled, Watcher assigns one process group to the attempt,
- macOS contains conforming non-detaching detectors. Watcher assigns one
process group to the attempt,
and the detector root plus every process it creates must remain in that group
until the attempt ends. Work moved outside the group is outside the cleanup
guarantee. Until issue #172's implementation, native conformance and review
are accepted, Watcher fails closed before detector launch on macOS and does
not claim macOS support.
guarantee. The runtime requires identity-verified sender readiness and
capability admission before releasing the detector.
- Every other host fails closed before detector launch until a documented and
tested containment contract is accepted for that host.

Expand Down Expand Up @@ -796,7 +795,8 @@ An older, newer or conformance-unvalidated version alone MUST NOT trigger a
warning. Native conformance remains an engineering coverage duty, not an
admission list or a claim that every version works. The earlier macOS 15 floor
and new-major admission wording are superseded; genuine safety checks remain.
This domain remains unshipped until the #172 product PR is accepted.
Source builds support this domain through the private per-attempt guard;
release-archive packaging and distribution remain separate.

An ordinary per-attempt private guard is Watcher runtime machinery, not a helper
service, only when it is unprivileged, ephemeral, shipped with the runtime,
Expand Down Expand Up @@ -1203,9 +1203,10 @@ count until an operator-approved offline vacuum. Permanent evidence grows with
events; bounded disposable history is not unlimited-duration total-size retention.

The initial operating target is 100 watches with default concurrency four.
Windows x86_64 is the initial distribution; Linux has the retained SQLite and
PostgreSQL automated proof scope, not an implied production soak. macOS remains
unsupported and fails closed before detector launch.
Windows x86_64 is the initial distribution. Linux and macOS have automated
SQLite/PostgreSQL lifecycle and health coverage; this is not a production
soak. macOS uses the non-detaching detector contract and a matching adjacent
guard in source builds; release-archive packaging remains separate.

### Snapshot authority and restore fencing

Expand Down
79 changes: 73 additions & 6 deletions docs/watcher-runtime.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,78 @@ remain historical evidence and are not accepted by this runtime.
Windows x86_64 is the initial supported Watcher distribution. Linux is covered
by the automated SQLite/PostgreSQL tests described below, with the same-build
process guard; that evidence does not substitute for a clean-host production
soak. This runtime delivery requires process containment on Windows and Linux.
macOS and other unclaimed hosts reject detector execution. macOS remains a
required Watcher product capability tracked in
[#165](https://github.com/lossyrob/telex/issues/165); this delivery does not claim
macOS containment.
soak. This runtime delivery requires process containment on every host that
runs detectors. Windows and Linux contain the whole detector process tree.
macOS contains the assigned process group of a trusted non-detaching detector
(see [macOS detector containment](#macos-detector-containment),
[#172](https://github.com/lossyrob/telex/issues/172)). Other hosts reject
detector execution. On macOS, the identity-verified `InstalledCurrent` Rust
client establishes send-only membership before detectors can launch. The same
SQLite/PostgreSQL lifecycle, recovery and health checks run on macOS, alongside
native containment conformance. Source builds require a matching
`watcher_process_guard` beside the runtime; the installation tool pins its
hash and verifies the pair. Release-archive packaging and distribution remain
separate from this source-build support.

## macOS detector containment

macOS has no ordinary-user primitive that contains a descendant after it
changes session or process group. Watcher therefore supports **trusted
non-detaching detectors** on macOS (ADR 0054, amended by ADR 0057): the
detector root and everything it starts must stay in the process group Watcher
assigns to the attempt. A detector that daemonizes, calls `setsid`, or moves
work to another group is nonconforming, and that work is outside the cleanup
guarantee.

Each attempt gets a private `watcher_process_guard` process, which must sit
beside the `telex-watcher` executable. The guard runs in its own session,
outside the detector group, and is controlled only through inherited private
handles. It also watches the runtime process itself through a kernel exit
notification, so runtime death ends the attempt even when another process
holds a copy of the runtime's control handle. It exits after writing the
attempt's receipt.

- **Admission.** Before launch, both the runtime and the guard require a
native (not Rosetta-translated) arm64 or x86_64 process on a release kernel,
and probe the nonreaping `waitid` and the process-group census on the actual
waiting leader. A failed or indeterminate check refuses the attempt with the
concrete reason. The OS version is never an admission input: there is no
minimum, maximum or tested-version allowlist.
- **Launch gate.** The guard forks a trampoline that becomes the group leader
and waits. The runtime durably records the identity (boot-session UUID,
guard and leader PID and start time, PGID, nonce, receipt path) and only then
sends the one-use release. If the guard or runtime is lost before release,
the detector never runs.
- **Cleanup.** The guard never reaps the leader while it may still signal, so
the PID and PGID cannot be reused under a signal. On exit, timeout,
cancellation, an output limit or runtime death it kills the group and
repeats until a fresh nonreaping wait plus a two-slot group census show only
the leader. It then gives up signal authority, reaps the leader, and requires
`kill(-PGID, 0)` to fail with `ESRCH` before it writes and syncs the receipt.
- **What Ended means.** The predicate above held. `proc_listpids` is a private
Apple ABI, guarded by the launch probe and native conformance tests rather
than a documented contract. ADR 0057 accepts one residual: a member caught
in a brief fork, exec or exit transition could be invisible to both checks,
leaving a stray copy of the trusted detector after Ended. No unrelated
process can be signalled in that case.
- **Unproven.** Guard loss after release, a census or wait error, or a former
group still visible after the reap leaves `orphan-containment-unproven`.
Recovery clears it only from the exact identity-matched receipt or a changed
boot-session UUID; a slow guard that finishes later is accepted on resume.
Watcher never signals a recorded PID during recovery.
- **Escaped pipe holders.** If the assigned group has ended but something
outside it still holds the detector's stdout or stderr, Watcher waits two
seconds for end-of-file, then closes its end and fails the attempt as
nonconforming. Containment for that attempt is still Ended; the escaped
process is neither signalled nor claimed. Its later writes fail.
- **Known limitation: leaked pipe ends.** macOS has no atomic close-on-exec
for pipes and sockets, so an unrelated program that the runtime starts at
the same moment can inherit a copy of an attempt's pipe ends. If it holds
the stdout or stderr write end, a conforming detector is reported as
nonconforming after the two-second wait (containment is still Ended). If it
holds the stdin write end, a detector that reads stdin to end-of-file waits
until its timeout. Neither case changes which processes are signalled or
what Ended means.

## Runtime failure diagnostics

Expand Down Expand Up @@ -95,7 +162,7 @@ Other native lock errors, including `EINVAL`/`ENOTSUP` on an OS or filesystem
that rejects OFD locking, fail startup with the OS error preserved and no
fallback. macOS 13 and earlier are not claimed for this mechanism. Linux and
other Unix targets keep `flock`; Windows keeps its existing `LockFileEx` byte
lock. Registry startup support does not enable macOS detector execution.
lock. Registry startup support is separate from macOS detector containment.

The returned `LocalManagement` handle sends bounded in-process requests to that
owner. The host exposes those same typed operations over a versioned same-user
Expand Down
16 changes: 8 additions & 8 deletions telex-watcher/installation.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ param(

Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
if (-not ($IsWindows -or $IsLinux)) { throw 'Watcher bootstrap supports Windows and Linux only.' }
if (-not ($IsWindows -or $IsLinux -or $IsMacOS)) { throw 'Watcher bootstrap supports Windows, Linux and macOS only.' }

function Assert-Absolute([string]$Path) {
if ([string]::IsNullOrWhiteSpace($Path) -or -not [IO.Path]::IsPathFullyQualified($Path)) {
Expand Down Expand Up @@ -55,7 +55,7 @@ function Assert-AuthorityAncestors([IO.DirectoryInfo]$Directory) {
} else {
$uid = & id -u
if ($LASTEXITCODE -ne 0) { throw 'Cannot determine current uid.' }
$owner = & stat -c '%u' -- $Directory.FullName
$owner = if ($IsMacOS) { & stat -f '%u' $Directory.FullName } else { & stat -c '%u' -- $Directory.FullName }
if ($LASTEXITCODE -ne 0 -or $owner -notin @('0', $uid)) {
throw "Installation authority has an untrusted ancestor: $Directory"
}
Expand Down Expand Up @@ -96,7 +96,7 @@ function Assert-Private([string]$Path) {
} else {
$uid = & id -u
if ($LASTEXITCODE -ne 0) { throw 'Cannot determine current uid.' }
$owner = & stat -c '%u' -- $Path
$owner = if ($IsMacOS) { & stat -f '%u' $Path } else { & stat -c '%u' -- $Path }
if ($LASTEXITCODE -ne 0 -or $owner -ne $uid) { throw "Installation authority has a different owner: $Path" }
$mode = [IO.File]::GetUnixFileMode($Path)
if (([int]$mode -band 63) -ne 0) { throw "Installation authority must be owner-only: $Path" }
Expand Down Expand Up @@ -169,13 +169,13 @@ function Assert-LaunchEnvironment($Record) {
}

function Get-ProcessGuardHash([string]$Watcher) {
if (-not $IsLinux) { return $null }
if (-not ($IsLinux -or $IsMacOS)) { return $null }
$guard = Join-Path (Split-Path -Parent $Watcher) 'watcher_process_guard'
if (-not (Test-Path -LiteralPath $guard -PathType Leaf)) {
throw 'Linux Watcher requires watcher_process_guard beside its retained executable.'
throw 'Unix Watcher requires watcher_process_guard beside its retained executable.'
}
if (([int][IO.File]::GetUnixFileMode($guard) -band 64) -eq 0) {
throw 'Linux watcher_process_guard must be executable by its owner.'
throw 'Unix watcher_process_guard must be executable by its owner.'
}
(Get-FileHash -LiteralPath $guard -Algorithm SHA256).Hash.ToLowerInvariant()
}
Expand Down Expand Up @@ -238,9 +238,9 @@ function Assert-Record($Record, [switch]$Installed) {
if ($Record.watcherSha256 -ne (Get-FileHash -LiteralPath $Record.watcherExecutable -Algorithm SHA256).Hash.ToLowerInvariant()) {
throw 'Watcher executable changed; create a new reviewed installation record.'
}
if ($IsLinux) {
if ($IsLinux -or $IsMacOS) {
if ($Record.processGuardSha256 -isnot [string] -or $Record.processGuardSha256 -cnotmatch '^[0-9a-f]{64}$') {
throw 'Invalid Linux process guard hash in installation record.'
throw 'Invalid Unix process guard hash in installation record.'
}
if ($Record.processGuardSha256 -cne (Get-ProcessGuardHash $Record.watcherExecutable)) {
throw 'Watcher process guard changed; create a new reviewed installation record.'
Expand Down
Loading
Loading