th-8e3087: SmoothFlow engine e2e — real daemon, fake-agent manifests, hooks contract, th flow

.changeset/smoothflow-engine-e2e.md

@@ -0,0 +1,7 @@
+---
+'@smooai/smooth': patch
+---
+
+SmoothFlow engine e2e suite (th-8e3087): `crates/smooth-daemon/tests/flow_e2e` boots a real `smooth-daemon` per test — isolated HOME, ephemeral port, private tmux server, no lock / tailscale / relay / credentials — and drives it over the flow WS, the HTTP siblings, the `th` binary and `POST /api/flow/hooks`, with `fake-agent` installed through harness manifests in four state-source flavours (hooks, learned id, native, scrape). 24 tests (~70 s): shell lifecycle, every agent transition, the permission long-poll, resume-on-death with `--resume`, the three-resume give-up, the duplicate-resume guard, the hooks contract per event, `th flow` / `th harness` JSON, the harness matrix + sort/hide prefs, `th harness add`, and proof the suite never touches the real `~/.smooth`. Runs on every PR (tmux installed, `SMOOTH_E2E_STRICT=1`); docs in `docs/Engineering/SmoothFlow-Testing.md`.
+
+Engine bugs the suite found and this release fixes: tmux under a non-UTF-8 locale (any daemon not started from a shell) rewrote the tab in the pane-dead / pane-size queries, so a dead pane was never detected and supervision was inert; `{daemon_url}` was rendered from a port-0 request address; a resumed or prompt-less harness stayed `starting` forever (`SessionStart` now makes a starting row idle); a `held` row was flapped by the supervisor. Rule 4's `held` was also racy end to end: `kill`, the supervision tick and `POST /api/flow/hooks` all read a row and then wrote it, so a hold written between one reader's read and its write was silently overwritten (~1 in 3 under load) — a dying agent's last `Stop` un-held the row and the next tick read its dead tmux session as a crash. All three now serialise on a per-session lock and re-read under it, and a hook naming a held row is ignored. The same read-before-write ordering also stranded a resumed row: `hook` read the row before `relaunch` wrote `starting`, saw the pre-kill `idle`, and `SessionStart`'s promote-a-starting-row rule declined, so a `kill --resume` left the row `starting` for good.

.github/workflows/pr-checks.yml

@@ -94,6 +94,10 @@ jobs:
               shell: bash
               run: echo "run=${{ github.event_name != 'pull_request' || steps.filter.outputs.rust == 'true' }}" >>"$GITHUB_OUTPUT"
 
+            # tmux: the SmoothFlow engine e2e suite (crates/smooth-daemon/tests/
+            # flow_e2e, th-8e3087) boots a real daemon per test and runs
+            # fake-agent under a private tmux server. Without it the suite
+            # skips — and SMOOTH_E2E_STRICT below turns that skip into a failure.
             - name: Install system dependencies
               if: steps.gate.outputs.run == 'true' && runner.os == 'Linux'
               # ubuntu-latest ships a google-chrome apt source that intermittently
@@ -103,7 +107,7 @@ jobs:
               # a mirror we don't use (pearl th-79cef7).
               run: |
                   sudo rm -f /etc/apt/sources.list.d/google-chrome*
-                  sudo apt-get update && sudo apt-get install -y libdbus-1-dev libcap-ng-dev pkg-config protobuf-compiler
+                  sudo apt-get update && sudo apt-get install -y libdbus-1-dev libcap-ng-dev pkg-config protobuf-compiler tmux
 
             # No apt on Windows — protoc is the only native build-time tool the
             # workspace needs there (tonic-build in smooth-scribe et al.).
@@ -165,6 +169,11 @@ jobs:
               run: cargo nextest run --profile ci ${{ matrix.test_args }}
               env:
                   NEXTEST_EXPERIMENTAL_LIBTEST_JSON: 1
+                  # The flow e2e suite must RUN on Linux, not skip: a missing
+                  # tmux/bash/curl/`th` fails the job instead of passing it
+                  # having proved nothing. Windows compiles the suite empty
+                  # (`#![cfg(unix)]`) — no tmux there.
+                  SMOOTH_E2E_STRICT: ${{ runner.os == 'Linux' && '1' || '0' }}
                   # Windows test threads get ~1MB of stack; parsing/dropping the
                   # full clap derive tree in a debug build overflows it as the
                   # `th` Command graph grows (th-6c4ddf, then th-9483e8). libtest

crates/smooth-daemon/examples/flow_e2e_server.rs

@@ -22,6 +22,12 @@ fn arg(args: &[String], name: &str) -> Option<String> {
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
+    // `RUST_LOG` (default `warn`) — the supervisor's trace lines are how a
+    // silent state machine gets debugged.
+    tracing_subscriber::fmt()
+        .with_env_filter(tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("warn")))
+        .with_writer(std::io::stderr)
+        .init();
     let args: Vec<String> = std::env::args().collect();
     let addr = arg(&args, "--addr").unwrap_or_else(|| "127.0.0.1:0".into());
     let workspace = PathBuf::from(arg(&args, "--workspace").unwrap_or_else(|| ".".into()));

crates/smooth-daemon/src/operator.rs

@@ -1071,6 +1071,13 @@ pub async fn serve_local_flavor(addr: SocketAddr) -> Result<()> {
     // the app bundle) funnels through here, so this is the choke point. Held
     // to shutdown; the OS releases it if we die (pearl th-c71e6f).
     let _instance = crate::single_instance::acquire_default().await?;
+    // Port 0 (an ephemeral port — what the e2e suites bind) has to be
+    // resolved BEFORE the server is built: `{daemon_url}` (the address a
+    // `th code` / fake-agent pane posts its hooks back to) is rendered from
+    // this value when the flow router is installed below, and the bound port
+    // is only known after `spawn()`. th-8e3087 caught every hook of a port-0
+    // daemon going to `http://127.0.0.1:0`.
+    let addr = resolve_ephemeral_port(addr)?;
     let token = provision_local_token()?;
     // The local flavor's tools: the workspace-confined fs/grep set + an
     // OS-sandboxed `bash` whose egress is routed through the goalie proxy (when
@@ -1363,6 +1370,19 @@ pub async fn serve_local_flavor(addr: SocketAddr) -> Result<()> {
     Ok(())
 }
 
+/// `addr` with a port of 0 replaced by a port the OS just handed out (bound
+/// and released — the same tiny race every "pick a free port" helper has).
+/// Any other port is returned unchanged.
+fn resolve_ephemeral_port(addr: SocketAddr) -> Result<SocketAddr> {
+    if addr.port() != 0 {
+        return Ok(addr);
+    }
+    let probe = std::net::TcpListener::bind(addr).with_context(|| format!("probing an ephemeral port on {addr}"))?;
+    let bound = probe.local_addr().context("reading the probed ephemeral port")?;
+    drop(probe);
+    Ok(bound)
+}
+
 /// The URL a process on this host reaches the daemon at (th-0f6126: the
 /// `{daemon_url}` a `th code` pane connects back to). An unspecified bind
 /// address is reachable on loopback.
@@ -2395,6 +2415,20 @@ mod tests {
         assert_eq!(std::fs::read_to_string(&path).unwrap(), "127.0.0.1:9999");
     }
 
+    /// th-8e3087: a port-0 bind is resolved to a real port BEFORE the flow
+    /// router renders `{daemon_url}` from it; fixed ports pass through.
+    #[test]
+    fn ephemeral_port_is_resolved_and_fixed_ports_pass_through() {
+        let fixed: SocketAddr = "127.0.0.1:8787".parse().unwrap();
+        assert_eq!(resolve_ephemeral_port(fixed).unwrap(), fixed);
+        let zero: SocketAddr = "127.0.0.1:0".parse().unwrap();
+        let resolved = resolve_ephemeral_port(zero).unwrap();
+        assert_ne!(resolved.port(), 0);
+        assert_eq!(resolved.ip(), zero.ip());
+        assert!(loopback_url(resolved).ends_with(&format!(":{}", resolved.port())));
+        assert_eq!(loopback_url("0.0.0.0:8787".parse().unwrap()), "http://127.0.0.1:8787");
+    }
+
     #[test]
     fn token_path_is_under_the_home_dot_smooth() {
         let p = token_path();

crates/smooth-daemon/tests/fixtures/fake-agent

@@ -0,0 +1,221 @@
+#!/usr/bin/env bash
+# fake-agent — a scripted coding-agent CLI for SmoothFlow e2e tests
+# (pearl th-8e3087). The engine launches it through a HARNESS MANIFEST
+# (tests/fixtures/harnesses/*.toml → ~/.smooth/harnesses/ in the test HOME),
+# exactly the way it launches claude / codex / opencode, and it behaves like a
+# real harness everywhere the engine can tell: argv, hook posts, the
+# permission long-poll, the usage-limit banner, exit codes. No LLM, no network
+# beyond the flow daemon. A superset of `fake-claude` (the macOS lane's stub).
+#
+# Contract (docs/Engineering/SmoothFlow-Testing.md):
+#   argv     `--session-id <id>` (preassigned), `--resume <id>` (resume=1),
+#            `--model <m>`; the first positional is a PROMPT: commands
+#            separated by `;`, run after the start-up script.
+#   daemon   `$SMOOTH_URL` (the manifest's `{daemon_url}`), else
+#            `./.flow-e2e-addr` in the cwd, else `$HOME/.smooth/daemon.addr`.
+#   mode     `$FAKE_AGENT_MODE`: hooks (default) — Claude Code's event names;
+#            native — th code's `turn_start`/`turn_end`/`ask`/`bye` (the
+#            manifest's event_map); scrape — posts NOTHING, paints the pane
+#            so the engine's scraper drives state.
+#   id       learned mode (no --session-id): `./.fake-agent-session-id` in
+#            the cwd, else `$FAKE_AGENT_SESSION_ID`, else a fresh uuid — the
+#            first hook binds it to the row by cwd, like opencode/codex.
+#   script   `./.fake-agent-script` — one command per line, run on EVERY
+#            start (fresh and resumed) before the prompt. How a test makes a
+#            relaunch crash again, or a resumed agent do work.
+#   stdin    one command per line (what `flow.send` pastes):
+#              /work <text>   a turn: working → idle
+#              /perm          ask permission; prints `decision: <reply>`
+#              /ask <text>    a question notification (needs_you · question)
+#              /limit [time]  the usage-limit banner (default `11:59pm`)
+#              /exit [code]   SessionEnd / bye, then exit (default 0)
+#              /crash [code]  exit with no hook at all (default 3)
+#              /sleep <secs>
+#              anything else  echoed back as `echo: <line>`
+#   log      `$FAKE_AGENT_LOG` (default `./.fake-agent.log`): one line per
+#            launch (`argv …`), hook (`hook <Event> → <reply>`) and command.
+#   screen   `fake-agent ready sid=<id> resume=<0|1> mode=<mode>` on boot
+#            (proves a relaunch used `--resume`), then `> ` after each line.
+set -u
+
+sid=""
+resume=0
+prompt=""
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --session-id) sid="$2"; shift 2 ;;
+        --resume) sid="$2"; resume=1; shift 2 ;;
+        --model) shift 2 ;;
+        --*) shift ;;
+        *) if [ -z "$prompt" ]; then prompt="$1"; fi; shift ;;
+    esac
+done
+
+mode="${FAKE_AGENT_MODE:-hooks}"
+harness="${FAKE_AGENT_HARNESS:-fake-agent}"
+log="${FAKE_AGENT_LOG:-./.fake-agent.log}"
+
+if [ -z "$sid" ]; then
+    if [ -r ./.fake-agent-session-id ]; then
+        sid="$(tr -d '[:space:]' <./.fake-agent-session-id)"
+    elif [ -n "${FAKE_AGENT_SESSION_ID:-}" ]; then
+        sid="$FAKE_AGENT_SESSION_ID"
+    else
+        sid="learned-$(date +%s)-$$"
+    fi
+fi
+
+addr="${SMOOTH_URL:-}"
+[ -n "$addr" ] || addr="$(cat ./.flow-e2e-addr 2>/dev/null || cat "$HOME/.smooth/daemon.addr" 2>/dev/null || true)"
+addr="$(printf '%s' "$addr" | tr -d '[:space:]')"
+case "$addr" in
+    http://* | https://*) ;;
+    *) addr="http://$addr" ;;
+esac
+addr="${addr%/}"
+
+logline() { printf '%s\n' "$*" >>"$log" 2>/dev/null || true; }
+logline "argv sid=$sid resume=$resume mode=$mode model_prompt=[$prompt]"
+
+# hook <Event> <payload-json> [timeout-s] — prints the daemon's reply body.
+# Scrape mode never posts: the engine must learn everything from the pane.
+hook() {
+    if [ "$mode" = scrape ]; then
+        return 0
+    fi
+    local reply
+    reply="$(curl -sS -m "${3:-5}" -X POST -H 'Content-Type: application/json' \
+        --data-binary "{\"harness\":\"$harness\",\"event\":\"$1\",\"session_id\":\"$sid\",\"cwd\":\"$PWD\",\"payload\":$2}" \
+        "$addr/api/flow/hooks" 2>/dev/null)"
+    logline "hook $1 → ${reply:-<none>}"
+    printf '%s' "$reply"
+}
+
+json_str() {
+    # Minimal JSON string escaping for the text we generate.
+    printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g'
+}
+
+# The scraper's "working" marker is painted on its own line and ERASED when
+# the turn ends, so the engine's last-12-lines rule sees an idle pane again.
+paint_working() { printf 'Thinking... (esc to interrupt)\n'; }
+erase_working() { printf '\033[1A\r\033[2K'; }
+
+run_cmd() {
+    local line="$1"
+    logline "cmd $line"
+    case "$line" in
+        /work*)
+            local text="${line#/work}"
+            text="${text# }"
+            case "$mode" in
+                native)
+                    hook turn_start '{}' >/dev/null
+                    sleep "${FAKE_AGENT_WORK_SECS:-0}"
+                    hook turn_end "{\"message\":\"done: $(json_str "$text")\"}" >/dev/null
+                    ;;
+                scrape)
+                    paint_working
+                    sleep "${FAKE_AGENT_WORK_SECS:-3}"
+                    erase_working
+                    ;;
+                *)
+                    hook UserPromptSubmit "{\"prompt\":\"$(json_str "$text")\"}" >/dev/null
+                    hook PreToolUse '{"tool_name":"Bash","tool_input":{"command":"echo hi"}}' >/dev/null
+                    sleep "${FAKE_AGENT_WORK_SECS:-0}"
+                    hook PostToolUse '{"tool_name":"Bash","tool_response":{"stdout":"hi"}}' >/dev/null
+                    hook Stop "{\"last_assistant_message\":\"done: $(json_str "$text")\"}" >/dev/null
+                    ;;
+            esac
+            printf 'worked: %s\n' "$text"
+            ;;
+        /perm*)
+            case "$mode" in
+                native)
+                    hook ask '{"reason":"permission","message":"run git push?"}' >/dev/null
+                    printf 'asked\n'
+                    ;;
+                scrape)
+                    printf 'Bash(git push)\nDo you want to proceed?\n > 1. Yes\n   2. Yes, and don'"'"'t ask again\n   3. No (esc)\n'
+                    local key
+                    IFS= read -r -s -n1 key || key=""
+                    case "$key" in
+                        1) printf 'decision: allow\n' ;;
+                        2) printf 'decision: allow_session\n' ;;
+                        *) printf 'decision: deny\n' ;;
+                    esac
+                    ;;
+                *)
+                    local reply
+                    reply="$(hook PermissionRequest '{"tool_name":"Bash","tool_input":{"command":"git push"}}' 130)"
+                    printf 'decision: %s\n' "$reply"
+                    ;;
+            esac
+            ;;
+        /ask*)
+            local text="${line#/ask}"
+            text="${text# }"
+            if [ "$mode" = native ]; then
+                hook ask "{\"reason\":\"question\",\"message\":\"$(json_str "$text")\"}" >/dev/null
+            else
+                hook Notification "{\"notification_type\":\"idle_prompt\",\"message\":\"$(json_str "$text")\"}" >/dev/null
+            fi
+            printf 'asked: %s\n' "$text"
+            ;;
+        /limit*)
+            local at="${line#/limit}"
+            at="${at# }"
+            printf "You've hit your usage limit. Your limit will reset at %s\n" "${at:-11:59pm}"
+            ;;
+        /exit*)
+            local code="${line#/exit}"
+            code="${code// /}"
+            if [ "$mode" = native ]; then
+                hook bye '{}' >/dev/null
+            else
+                hook SessionEnd '{"reason":"exit"}' >/dev/null
+            fi
+            logline "exit ${code:-0}"
+            exit "${code:-0}"
+            ;;
+        /crash*)
+            local code="${line#/crash}"
+            code="${code// /}"
+            logline "crash ${code:-3}"
+            exit "${code:-3}"
+            ;;
+        /sleep*)
+            local secs="${line#/sleep}"
+            sleep "${secs// /}"
+            ;;
+        *)
+            printf 'echo: %s\n' "$line"
+            ;;
+    esac
+}
+
+if [ "$mode" = native ]; then
+    : # th code has no SessionStart; its first report is turn_start.
+else
+    hook SessionStart "{\"source\":\"$([ "$resume" = 1 ] && echo resume || echo startup)\"}" >/dev/null
+fi
+printf 'fake-agent ready sid=%s resume=%s mode=%s\n' "$sid" "$resume" "$mode"
+
+if [ -r ./.fake-agent-script ]; then
+    while IFS= read -r line || [ -n "$line" ]; do
+        run_cmd "$line"
+    done <./.fake-agent-script
+fi
+if [ -n "$prompt" ]; then
+    IFS=';' read -r -a parts <<<"$prompt"
+    for part in "${parts[@]}"; do
+        run_cmd "${part# }"
+    done
+fi
+
+printf '> '
+while IFS= read -r line; do
+    run_cmd "$line"
+    printf '> '
+done
+logline "stdin closed"

crates/smooth-daemon/tests/fixtures/harnesses/fake-agent-learned.toml

@@ -0,0 +1,43 @@
+# fake-agent-learned — hooks state, session id LEARNED from the first hook
+# (the opencode / codex shape): no `--session-id`; the agent reports the id it
+# picked (./.fake-agent-session-id in the cwd, else a fresh one) and the engine
+# binds it to the newest id-less row in that worktree.
+name = "fake-agent-learned"
+display_name = "Fake Agent (learned id)"
+
+[binary]
+names = ["fake-agent"]
+prefer_paths = [".local/bin/fake-agent"]
+
+[launch]
+argv = ["--model", "{model}", "{prompt}"]
+prompt_as = "argv"
+session_id = "learned"
+
+[launch.env]
+SMOOTH_URL = "{daemon_url}"
+FAKE_AGENT_MODE = "hooks"
+
+[resume]
+argv = ["--resume", "{session_id}"]
+mode = "resume_session"
+
+[state]
+source = "hooks"
+
+[state.hooks]
+install = "none — fake-agent posts to /api/flow/hooks itself"
+
+[state.scrape]
+working = ["esc to interrupt"]
+idle = ["(?m)^> ?$"]
+needs_you = ["do you want to proceed"]
+usage_limit = ["usage limit", "limit will reset"]
+
+[steer]
+method = "bracketed_paste"
+submit_key = "Enter"
+
+[kill]
+signal = "TERM"
+grace_ms = 1000