Skip to content

Go-Standalone spawn hardening: never park the window on a failed spawn (#781) #787

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
aarontrowbridge wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Go-Standalone spawn hardening: never park the window on a failed spawn (#781) #787

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
233 changes: 153 additions & 80 deletions packages/extension/src/extension.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,13 @@ import { probeCommand, formatHealthReport, probeOpencodeTui, type HealthResult }
import { fleetHealthReport, FLEET_GUARD_REL } from "./fleet_health";
import { isFleetClient, getFleetRole, goStandalone, readFleetConfig, migrateLegacyFallback } from "./fleet_fallback";
import { resolveHubTarget, restartHub } from "./hub_ops";
import {
goStandaloneSpawn,
journalSelfHeal,
localDbPath,
OfferGate,
ReofferBackoff,
} from "./standalone_spawn";
import { registerAmicodeTerminal } from "./terminal";
import { amicodeServiceDisposal, startAmicodeService } from "./amicode_service_wiring";
import { registerOpencodeUpdater } from "./opencode_updater_wiring";
Expand Down Expand Up @@ -103,6 +110,22 @@ let distillerSetup: DistillerSetup | undefined;
let devicePollTimer: ReturnType<typeof setInterval> | undefined;
/** Fleet client tunnel poll — when the machine is a fleet client (guard `exit 1`), we don't spawn. */
let fleetClientPoll: ReturnType<typeof setInterval> | undefined;
/** Restarts the attach poll after a failed Go-Standalone spawn (#781 AC1) —
* set by the client-mode boot; undefined when this window never polled. */
let fleetPollResume: (() => void) | undefined;
/** True while a Go-Standalone spawn attempt is in flight — the attach loop
* holds its re-offers so no banner fires mid-spawn (#781 AC4). */
let standaloneAttemptActive = false;

/** Stop the fleet client attach poll (idempotent). The poll stops ONLY via the
* FleetPollGuard path (#781: after the local server is healthy) or disposal —
* never before a spawn attempt. */
function stopFleetClientPoll(): void {
if (fleetClientPoll) {
clearInterval(fleetClientPoll);
fleetClientPoll = undefined;
}
}

const DEVICE_POLL_MS = 2500; // mirror the RunsManager cadence

Expand Down Expand Up @@ -610,7 +633,25 @@ export async function activate(ctx: vscode.ExtensionContext): Promise<void> {
// Poll the tunnel — when the canonical server is reachable the forward answers 200.
let fleetReady = false;
let fleetChecks = 0;
let fleetNotified = false;
// #781 AC2: the one-shot `fleetNotified` flag is gone. Re-offers now follow
// a per-window exponential backoff (base 30s, ceiling 5 min) that resets on
// any successful attach, and an OfferGate keeps exactly one banner alive so
// offers can never stack into double banners (AC4).
const reoffer = new ReofferBackoff();
const offerGate = new OfferGate();
const offerStandalone = (reason: string) => {
if (standaloneAttemptActive) return; // a spawn attempt is in flight — hold offers
if (!reoffer.due() || !offerGate.tryBegin()) return;
reoffer.recordOffer();
opencodeChannel.appendLine(`[fleet] ${reason} (checks=${fleetChecks}) — offering standalone`);
void vscode.window
.showWarningMessage(`Amicode: fleet tunnel down — canonical unreachable. Go standalone?`, `Go Standalone`, `Show log`)
.then((pick) => {
offerGate.end();
if (pick === `Go Standalone`) void vscode.commands.executeCommand(`amicode.fleet.goStandalone`);
else if (pick === `Show log`) opencodeChannel.show();
});
};
const checkFleet = async () => {
fleetChecks++;
try {
Expand All @@ -621,7 +662,7 @@ export async function activate(ctx: vscode.ExtensionContext): Promise<void> {
const up = r.ok || (r.status >= 200 && r.status < 400);
if (up && !fleetReady) {
fleetReady = true;
fleetNotified = false;
reoffer.reset(); // a successful attach restarts the re-offer schedule (AC2)
opencodeReadyUrl = new URL(`http://127.0.0.1:${fleetPort}`);
statusBar?.setServerReady(true);
sseClient?.connect(opencodeReadyUrl);
Expand All @@ -641,15 +682,8 @@ export async function activate(ctx: vscode.ExtensionContext): Promise<void> {
opencodeChannel.appendLine(`[fleet] waiting for tunnel 127.0.0.1:${fleetPort} — canonical unreachable, will retry`);
}
// After ~10s (5 checks) still down → offer standalone visibly, not just a log
if (!up && !fleetReady && !fleetNotified && fleetChecks >= 5) {
fleetNotified = true;
opencodeChannel.appendLine(`[fleet] tunnel still down after ${fleetChecks} checks — offering standalone`);
void vscode.window
.showWarningMessage(`Amicode: fleet tunnel down — canonical unreachable. Go standalone?`, `Go Standalone`, `Show log`)
.then((pick) => {
if (pick === `Go Standalone`) void vscode.commands.executeCommand(`amicode.fleet.goStandalone`);
else if (pick === `Show log`) opencodeChannel.show();
});
if (!up && !fleetReady && fleetChecks >= 5) {
offerStandalone(`tunnel still down after ${fleetChecks} checks — offering standalone`);
}
} catch {
if (fleetReady) {
Expand All @@ -660,20 +694,17 @@ export async function activate(ctx: vscode.ExtensionContext): Promise<void> {
} else if (fleetChecks === 1) {
opencodeChannel.appendLine(`[fleet] waiting for tunnel 127.0.0.1:${fleetPort} — will retry`);
}
if (!fleetReady && !fleetNotified && fleetChecks >= 5) {
fleetNotified = true;
opencodeChannel.appendLine(`[fleet] tunnel still down after ${fleetChecks} checks — offering standalone`);
void vscode.window
.showWarningMessage(`Amicode: fleet tunnel down — canonical unreachable. Go standalone?`, `Go Standalone`, `Show log`)
.then((pick) => {
if (pick === `Go Standalone`) void vscode.commands.executeCommand(`amicode.fleet.goStandalone`);
else if (pick === `Show log`) opencodeChannel.show();
});
if (!fleetReady && fleetChecks >= 5) {
offerStandalone(`tunnel still down after ${fleetChecks} checks — offering standalone`);
}
}
};
fleetClientPoll = setInterval(() => void checkFleet(), 2000);
ctx.subscriptions.push({ dispose: () => { if (fleetClientPoll) clearInterval(fleetClientPoll); fleetClientPoll = undefined; } });
const startFleetPoll = () => {
if (!fleetClientPoll) fleetClientPoll = setInterval(() => void checkFleet(), 2000);
};
fleetPollResume = startFleetPoll; // a failed Go-Standalone spawn restores the poll (#781 AC1)
startFleetPoll();
ctx.subscriptions.push({ dispose: () => stopFleetClientPoll() });
void checkFleet();
// Fallback status bar already handles the fallback-active case; in pure
// client mode we surface tunnel health via the fleet health warning above.
Expand Down Expand Up @@ -1372,65 +1403,110 @@ export async function activate(ctx: vscode.ExtensionContext): Promise<void> {
}
refreshFleetStatus();
opencodeChannel.appendLine(`[fleet] Go Standalone — restarting server locally (was binary=${prevBinary || "(vendored)"} port=${prevPort})`);
// #781: the fleet attach poll is NOT stopped before the spawn. It stops
// exactly once the local server reports healthy, and every failure path
// restores it (FleetPollGuard inside goStandaloneSpawn) — a failed spawn
// can never leave the window dead (AC1), and re-offers resume under the
// attach loop's per-window backoff (AC2).
standaloneAttemptActive = true; // hold the attach loop's re-offers during the attempt (AC4)
const runSqlite3 = (cmd: string[]): Promise<{ code: number; stdout: string; stderr: string }> =>
new Promise((resolve) => {
const { execFile } = require("node:child_process") as typeof import("node:child_process");
execFile(cmd[0]!, cmd.slice(1), { timeout: 10_000, encoding: "utf8" }, (error, stdout, stderr) => {
resolve({
code: error ? (typeof error.code === "number" ? error.code : 1) : 0,
stdout: String(stdout ?? ""),
stderr: String(stderr ?? ""),
});
});
});
try {
await serverManager?.stop();
statusBar?.setServerReady(false);
opencodeReadyUrl = undefined;
// Stop the fleet client poll if running
if (fleetClientPoll) { clearInterval(fleetClientPoll); fleetClientPoll = undefined; }
// Spawn a fresh local server (selected harness, fresh resolution — the
// standalone path deliberately re-resolves, ephemeral port)
const standaloneCfg = vscode.workspace.getConfiguration("amicode");
const standaloneLaunch = resolveSelectedLaunch({
harnessId: standaloneCfg.get<string>("harness", "opencode"),
opencodeBinary: "", // standalone drops the override by design
telaioBinary: standaloneCfg.get<string>("telaioBinary", ""),
telaioAppDir: standaloneCfg.get<string>("telaioAppDir", ""),
extensionPath: ctx.extensionPath,
});
harnessEnvCurrent = standaloneLaunch.descriptor.spawnEnvAdditions({
telaioAppDir: standaloneCfg.get<string>("telaioAppDir", ""),
});
harnessConsumesOpencodeConfig = standaloneLaunch.descriptor.consumesOpencodeConfig;
const amicoRunBinDir2 = resolveAmicoRunBinDir(ctx.extensionPath);
const freshManager = new ServerManager({
binary: standaloneLaunch.binary,
cwd: opencodeProject.projectDir,
port: undefined, // ephemeral — standalone
env: spawnEnv({
amicoRunBinDir: amicoRunBinDir2,
serverPassword,
configContent: buildOpencodeConfigContent(
opencodeProject.agentsPath,
opencodeProject.templatePath,
runsRoot,
undefined,
undefined,
opencodeProject.skillPaths,
opencodeProject.skillsStageDir,
opencodeProject.vaultDir,
opencodeProject.mounts,
validatedModelPin(vscode.workspace.getConfiguration("amicode").get<string>("defaultModel", "").trim() || resolveModelPin()),
telemetryOpen(),
),
}),
channel: opencodeChannel,
});
serverManager = freshManager;
ctx.subscriptions.push({ dispose: () => void freshManager.stop() });
freshManager.onReady((url) => {
opencodeReadyUrl = url;
statusBar?.setServerReady(true);
sseClient?.connect(url);
if (vscode.workspace.getConfiguration("amicode").get<boolean>("chat.autoOpen", true)) {
ChatPanel.openOrReveal(ctx, url, serverAuthToken(serverPassword), opencodeProject.projectDir);
}
const result = await goStandaloneSpawn({
poll: { stopPoll: () => stopFleetClientPoll(), resumePoll: () => fleetPollResume?.() },
startServer: async () => {
// Spawn a fresh local server (selected harness, fresh resolution — the
// standalone path deliberately re-resolves, ephemeral port)
const standaloneCfg = vscode.workspace.getConfiguration("amicode");
const standaloneLaunch = resolveSelectedLaunch({
harnessId: standaloneCfg.get<string>("harness", "opencode"),
opencodeBinary: "", // standalone drops the override by design
telaioBinary: standaloneCfg.get<string>("telaioBinary", ""),
telaioAppDir: standaloneCfg.get<string>("telaioAppDir", ""),
extensionPath: ctx.extensionPath,
});
harnessEnvCurrent = standaloneLaunch.descriptor.spawnEnvAdditions({
telaioAppDir: standaloneCfg.get<string>("telaioAppDir", ""),
});
harnessConsumesOpencodeConfig = standaloneLaunch.descriptor.consumesOpencodeConfig;
const amicoRunBinDir2 = resolveAmicoRunBinDir(ctx.extensionPath);
const freshManager = new ServerManager({
binary: standaloneLaunch.binary,
cwd: opencodeProject.projectDir,
port: undefined, // ephemeral — standalone
env: spawnEnv({
amicoRunBinDir: amicoRunBinDir2,
serverPassword,
configContent: buildOpencodeConfigContent(
opencodeProject.agentsPath,
opencodeProject.templatePath,
runsRoot,
undefined,
undefined,
opencodeProject.skillPaths,
opencodeProject.skillsStageDir,
opencodeProject.vaultDir,
opencodeProject.mounts,
validatedModelPin(vscode.workspace.getConfiguration("amicode").get<string>("defaultModel", "").trim() || resolveModelPin()),
telemetryOpen(),
),
}),
channel: opencodeChannel,
});
serverManager = freshManager;
ctx.subscriptions.push({ dispose: () => void freshManager.stop() });
freshManager.onReady((url) => {
opencodeReadyUrl = url;
statusBar?.setServerReady(true);
sseClient?.connect(url);
if (vscode.workspace.getConfiguration("amicode").get<boolean>("chat.autoOpen", true)) {
ChatPanel.openOrReveal(ctx, url, serverAuthToken(serverPassword), opencodeProject.projectDir);
}
});
await freshManager.start();
},
notice: (n) => Promise.resolve(vscode.window.showErrorMessage(n.title, ...n.actions)),
confirmHeal: async () =>
(await vscode.window.showWarningMessage(
"Run the local-DB journal self-heal? This rewrites local session data (a backup of the DB is kept first).",
{ modal: true },
"Self-heal",
"Cancel",
)) === "Self-heal",
heal: () =>
journalSelfHeal({
dbPath: localDbPath(),
run: runSqlite3,
fs: { existsSync: (p) => fs.existsSync(p), copyFileSync: (src, dest) => fs.copyFileSync(src, dest) },
}),
onHealResult: (r) => {
const detail = r.steps.map((s) => `${s.step}: ${s.detail}`).join(" | ");
opencodeChannel.appendLine(`[fleet] journal self-heal ok=${r.ok} — ${detail}`);
if (r.ok) {
void vscode.window.showInformationMessage("Amicode: journal self-heal applied — try Go Standalone again.");
} else {
void vscode.window.showErrorMessage(`Amicode: journal self-heal failed — ${detail}`);
}
},
log: (line) => opencodeChannel.appendLine(line),
});
await freshManager.start();
void vscode.window.showInformationMessage("Amicode: Standalone mode — running locally. Your local sessions are now visible.");
} catch (e) {
void vscode.window.showErrorMessage(`Amicode: go standalone failed — ${(e as Error).message}`);
opencodeChannel.appendLine(`[fleet] go standalone failed: ${(e as Error).message}`);
if (result.ok) {
void vscode.window.showInformationMessage("Amicode: Standalone mode — running locally. Your local sessions are now visible.");
}
} finally {
standaloneAttemptActive = false;
}
};

Expand Down Expand Up @@ -1958,10 +2034,7 @@ export async function activate(ctx: vscode.ExtensionContext): Promise<void> {
}

export function deactivate(): void {
if (fleetClientPoll) {
clearInterval(fleetClientPoll);
fleetClientPoll = undefined;
}
stopFleetClientPoll();
if (devicePollTimer) {
clearInterval(devicePollTimer);
devicePollTimer = undefined;
Expand Down
51 changes: 46 additions & 5 deletions packages/extension/src/server_manager.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,14 @@ export interface ServerOptions {
channel: vscode.OutputChannel;
/** Fixed port to serve on. 0 (default) picks a free ephemeral port each start. */
port?: number;
/** Health-probe budget in ms. Default 30_000; tests inject a short one. */
healthTimeoutMs?: number;
}

/** Tail length of the child's captured output attached to a failed start
* (#781) — long enough to carry a crash cause, short enough for a toast. */
const SPAWN_OUTPUT_TAIL = 4000;

export class ServerManager {
private child?: cp.ChildProcessByStdio<null, Readable, Readable>;
private _port?: number;
Expand Down Expand Up @@ -77,8 +83,22 @@ export class ServerManager {
});
this.child = child;

child.stdout.on("data", (b: Buffer) => this.opts.channel.append(`[opencode] ${b.toString()}`));
child.stderr.on("data", (b: Buffer) => this.opts.channel.append(`[opencode!] ${b.toString()}`));
// Capture the child's output while spawning (#781): on a failed start the
// thrown error carries a tail of it, so the caller can match a known crash
// signature (e.g. the local-DB `duplicate column name` migration failure)
// instead of the generic timeout message hiding the cause.
let output = "";
const capture = (b: Buffer) => {
output = (output + b.toString()).slice(-SPAWN_OUTPUT_TAIL);
};
child.stdout.on("data", (b: Buffer) => {
capture(b);
this.opts.channel.append(`[opencode] ${b.toString()}`);
});
child.stderr.on("data", (b: Buffer) => {
capture(b);
this.opts.channel.append(`[opencode!] ${b.toString()}`);
});
child.on("exit", (code, signal) => {
this.opts.channel.appendLine(`[server] opencode exited code=${code} signal=${signal}`);
this._ready = false;
Expand All @@ -90,11 +110,32 @@ export class ServerManager {
// a healthy boot would read as a 30s timeout. Derived from the same env
// the child gets, so probe and server can never disagree.
const password = this.opts.env.OPENCODE_SERVER_PASSWORD;
const ready = await waitForHealth(`http://127.0.0.1:${port}/`, 30_000, password ? serverAuthHeader(password) : undefined);
const healthTimeoutMs = this.opts.healthTimeoutMs ?? 30_000;
// Fail fast on a crashed child (#781): a binary that dies during startup
// must surface immediately with its captured output — not burn the full
// health budget and throw a generic timeout. `close` (not `exit`) fires
// after the stdio streams drain, so the captured tail is complete.
const childClosed = new Promise<boolean>((resolve) => child.once("close", () => resolve(true)));
const healthSettled = waitForHealth(`http://127.0.0.1:${port}/`, healthTimeoutMs, password ? serverAuthHeader(password) : undefined).then(() => false);
const crashed = await Promise.race([healthSettled, childClosed]);
if (crashed) {
this.opts.channel.appendLine(`[server] opencode exited before becoming healthy — aborting start`);
this.stop();
const err = new Error(
"opencode exited during startup before becoming healthy — check the 'Amicode — opencode' output channel",
) as Error & { spawnOutput?: string };
err.spawnOutput = output;
throw err;
}
const ready = await healthSettled.then(() => true);
if (!ready) {
this.opts.channel.appendLine(`[server] opencode did not become healthy within 30s`);
this.opts.channel.appendLine(`[server] opencode did not become healthy within ${Math.round(healthTimeoutMs / 1000)}s`);
this.stop();
throw new Error("opencode failed to start within 30s — check the 'Amicode — opencode' output channel");
const err = new Error(
`opencode failed to start within ${Math.round(healthTimeoutMs / 1000)}s — check the 'Amicode — opencode' output channel`,
) as Error & { spawnOutput?: string };
err.spawnOutput = output;
throw err;
}
this._ready = true;
const url = new URL(`http://127.0.0.1:${port}`);
Expand Down
Loading
Loading