Skip to content

fix(tracing): record session outcomes on root spans #87

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Hesham Salman (Iron-Ham) wants to merge 1 commit into from
Closed

fix(tracing): record session outcomes on root spans #87

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions bt-daemon/src/translate/claude.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,7 @@ struct ClaudeTranslator {
git: Arc<GitMetadataCache>,
current_cwd: Option<String>,
last_turn_cwd: Option<String>,
last_turn_output: Option<Value>,
last_ts_ms: i64,
}

Expand Down Expand Up @@ -302,6 +303,7 @@ impl ClaudeTranslator {
git,
current_cwd: None,
last_turn_cwd: None,
last_turn_output: None,
last_ts_ms: 0,
}
}
Expand Down Expand Up @@ -842,15 +844,19 @@ impl ClaudeTranslator {
"Turn ended before tool completion",
ops,
);
let output = event
.payload
.get("last_assistant_message")
.cloned()
.or_else(|| event.payload.get("output").cloned());
if output.as_ref().is_some_and(|output| !output.is_null()) {
self.last_turn_output = output.clone();
Comment on lines +852 to +853

@chatgpt-codex-connector chatgpt-codex-connector Bot Sep 16, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Refresh Claude outcomes from late transcript rows

When the Stop hook supplies an empty last_assistant_message because the final assistant row is written afterward, this condition records "" as the session outcome. The existing claude_groups_streamed_rows_and_reads_late_final_output_at_session_end flow demonstrates that SessionEnd can subsequently read the real "done" output, but emit_main never refreshes last_turn_output, so the root closes with an empty output instead of the actual answer. Derive or update the tracked outcome when those late transcript rows are consumed.

Useful? React with 👍 / 👎.

}
ops.push(SpanOp::Merge(SpanRow {
span_id: turn_id.clone(),
root_span_id: self.root_span_id.clone(),
end_ms: Some(event.ts_ms),
output: event
.payload
.get("last_assistant_message")
.cloned()
.or_else(|| event.payload.get("output").cloned()),
output,
error,
..Default::default()
}));
Expand Down Expand Up @@ -920,6 +926,7 @@ impl ClaudeTranslator {
span_id: self.session_span_id.clone(),
root_span_id: self.root_span_id.clone(),
end_ms: Some(event.ts_ms),
output: self.last_turn_output.clone(),
..Default::default()
}));
}
Expand Down Expand Up @@ -1064,6 +1071,7 @@ impl AgentTranslator for ClaudeTranslator {
span_id: self.session_span_id.clone(),
root_span_id: self.root_span_id.clone(),
end_ms: Some(end_ms),
output: self.last_turn_output.clone(),
..Default::default()
}));
}
Expand Down
13 changes: 11 additions & 2 deletions bt-daemon/src/translate/codex.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,7 @@ struct Scope {
open_llm: Option<OpenLlm>,
open_tools: HashMap<String, (String, String)>, // call_id -> (tool span_id, turn_id)
last_turn_end_ms: Option<i64>,
last_turn_output: Option<Value>,
turn_seq: u32,
// Subagent-only:
agent_id: Option<String>,
Expand Down Expand Up @@ -1206,6 +1207,9 @@ impl CodexTranslator {
let output = str_field(payload, "last_agent_message")
.or_else(|| str_field(payload, "last_assistant_message"))
.map(|s| json!(s));
if let Some(output) = &output {
scope.last_turn_output = Some(output.clone());
}
ops.push(SpanOp::Merge(SpanRow {
span_id: turn.span_id,
root_span_id: self.root_span_id.clone(),
Expand Down Expand Up @@ -1326,16 +1330,20 @@ impl CodexTranslator {
if !self.root_opened {
return;
}
let end_ms = self
let main_scope = self
.main_path
.as_ref()
.and_then(|path| self.scopes.get(path))
.and_then(|path| self.scopes.get(path));
let end_ms = main_scope
.and_then(|scope| scope.last_turn_end_ms)
.unwrap_or(fallback_ts);
// The session outcome is the last completed turn's output.
let output = main_scope.and_then(|scope| scope.last_turn_output.clone());
ops.push(SpanOp::Merge(SpanRow {
span_id: self.root_span_id.clone(),
root_span_id: self.root_span_id.clone(),
end_ms: Some(end_ms),
output,
late_merge_key: Some(format!("session:stop:{end_ms}")),
..Default::default()
}));
Expand Down Expand Up @@ -1471,6 +1479,7 @@ impl Scope {
open_llm: None,
open_tools: HashMap::new(),
last_turn_end_ms: None,
last_turn_output: None,
turn_seq: 0,
agent_id: None,
agent_type: None,
Expand Down
15 changes: 15 additions & 0 deletions bt-daemon/src/translate/pi.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ impl TranslatorFactory for PiTranslatorFactory {
external_parent: None,
opened: false,
turn: None,
turn_output: None,
last_turn_output: None,
turn_seq: 0,
llm_seq: 0,
total_tools: 0,
Expand Down Expand Up @@ -309,6 +311,8 @@ struct PiTranslator {
external_parent: Option<String>,
opened: bool,
turn: Option<(String, Value)>,
turn_output: Option<Value>,
last_turn_output: Option<Value>,
turn_seq: u32,
llm_seq: u32,
total_tools: u32,
Expand Down Expand Up @@ -667,6 +671,11 @@ impl PiTranslator {
.error_message
.clone()
.filter(|_| matches!(message.stop_reason.as_deref(), Some("error" | "aborted")));
// Turn and root outcomes come from the latest completed assistant
// message; errored or aborted messages are not outcomes.
if error.is_none() {
self.turn_output = Some(output.clone());
Comment on lines +676 to +677

@chatgpt-codex-connector chatgpt-codex-connector Bot Sep 16, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Clear Pi turn outcomes when the final message fails

When a Pi turn emits a successful intermediate assistant message (for example, a tool call) and its later assistant message ends with stopReason: "error" or "aborted", this branch merely skips replacing turn_output; it leaves the intermediate message intact. close_turn then promotes that stale value to both the failed turn and root, making the failed session appear to have a completed outcome. Clear the current turn outcome on an error or abort, and base that decision on the stop reason even when errorMessage is absent.

Useful? React with 👍 / 👎.

}
let ttft = pending
.first_token_ms
.map(|first| (first - pending.start_ms) as f64 / 1000.0);
Expand Down Expand Up @@ -842,10 +851,15 @@ impl PiTranslator {
let Some((id, _)) = turn else {
return vec![];
};
let output = self.turn_output.take();
if output.is_some() {
self.last_turn_output = output.clone();
}
vec![SpanOp::Merge(SpanRow {
span_id: id,
root_span_id: self.effective_root_span_id.clone(),
end_ms: Some(ts),
output,
error,
..Default::default()
})]
Expand All @@ -859,6 +873,7 @@ impl PiTranslator {
span_id: self.root_span_id.clone(),
root_span_id: self.effective_root_span_id.clone(),
end_ms: Some(ts),
output: self.last_turn_output.take(),
metadata: Some(
json!({"total_turns":self.turn_seq,"total_tool_calls":self.total_tools}),
),
Expand Down
164 changes: 164 additions & 0 deletions bt-daemon/tests/claude_translator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,170 @@ fn claude_passive_hooks_do_not_create_blank_session_traces() {
assert_eq!(metadata["model"], "claude-test");
}

#[test]
fn claude_root_records_last_turn_output_on_session_end() {
let registry = Registry::default_agents();
let mut translator = registry.create("claude-code", "outcome-session");
let ctx = SessionCtx {
session_id: "outcome-session".into(),
config: None,
};
let event = |name: &str, ts_ms: i64, payload: Value| Envelope {
source: "claude-code".into(),
source_version: None,
plugin_version: None,
session_id: "outcome-session".into(),
event: name.into(),
ts_ms,
managed_run_id: None,
payload,
route: None,
config: None,
capture: None,
};

let mut ops = Vec::new();
for event in [
event(
"SessionStart",
1,
json!({"cwd":"/workspace/demo", "source":"startup", "model":"claude-test"}),
),
event(
"UserPromptSubmit",
2,
json!({"cwd":"/workspace/demo", "prompt":"ship it"}),
),
event(
"Stop",
3,
json!({"cwd":"/workspace/demo", "last_assistant_message":"Done. Shipped it."}),
),
event("SessionEnd", 4, json!({"cwd":"/workspace/demo"})),
] {
ops.extend(translator.handle(&event, &ctx).unwrap());
}

let rows = reduce(ops);
let turn = rows
.values()
.find(|row| row.name == "Turn 1")
.expect("turn span");
assert_eq!(turn.output, Some(json!("Done. Shipped it.")));
let root = rows
.values()
.find(|row| row.name == "Claude Code: demo")
.expect("root span");
assert_eq!(
root.output,
Some(json!("Done. Shipped it.")),
"the session outcome must land on the root span"
);
}

#[test]
fn claude_root_records_last_turn_output_on_finalize() {
let registry = Registry::default_agents();
let mut translator = registry.create("claude-code", "finalize-session");
let ctx = SessionCtx {
session_id: "finalize-session".into(),
config: None,
};
let event = |name: &str, ts_ms: i64, payload: Value| Envelope {
source: "claude-code".into(),
source_version: None,
plugin_version: None,
session_id: "finalize-session".into(),
event: name.into(),
ts_ms,
managed_run_id: None,
payload,
route: None,
config: None,
capture: None,
};

let mut ops = Vec::new();
for event in [
event(
"SessionStart",
1,
json!({"cwd":"/workspace/demo", "source":"startup", "model":"claude-test"}),
),
event(
"UserPromptSubmit",
2,
json!({"cwd":"/workspace/demo", "prompt":"ship it"}),
),
event(
"Stop",
3,
json!({"cwd":"/workspace/demo", "last_assistant_message":"Done. Shipped it."}),
),
] {
ops.extend(translator.handle(&event, &ctx).unwrap());
}
// No SessionEnd hook: the daemon finalizes the session after a teardown.
ops.extend(translator.finalize(&ctx).unwrap());

let rows = reduce(ops);
let root = rows
.values()
.find(|row| row.name == "Claude Code: demo")
.expect("root span");
assert_eq!(root.output, Some(json!("Done. Shipped it.")));
}

#[test]
fn claude_root_output_stays_unset_without_a_completed_turn() {
let registry = Registry::default_agents();
let mut translator = registry.create("claude-code", "no-turn-session");
let ctx = SessionCtx {
session_id: "no-turn-session".into(),
config: None,
};
let event = |name: &str, ts_ms: i64, payload: Value| Envelope {
source: "claude-code".into(),
source_version: None,
plugin_version: None,
session_id: "no-turn-session".into(),
event: name.into(),
ts_ms,
managed_run_id: None,
payload,
route: None,
config: None,
capture: None,
};

let mut ops = Vec::new();
for event in [
event(
"SessionStart",
1,
json!({"cwd":"/workspace/demo", "source":"startup", "model":"claude-test"}),
),
event(
"UserPromptSubmit",
2,
json!({"cwd":"/workspace/demo", "prompt":"interrupted"}),
),
event("SessionEnd", 3, json!({"cwd":"/workspace/demo"})),
] {
ops.extend(translator.handle(&event, &ctx).unwrap());
}

let rows = reduce(ops);
let root = rows
.values()
.find(|row| row.name == "Claude Code: demo")
.expect("root span");
assert_eq!(
root.output, None,
"a session with no completed turn has no outcome to record"
);
}

#[test]
fn claude_subagent_routing_tolerates_malformed_optional_metadata() {
let registry = Registry::default_agents();
Expand Down
39 changes: 39 additions & 0 deletions bt-daemon/tests/codex_translator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,45 @@ fn codex_happy_path_builds_session_turn_llm_tool_tree() {
);
}

#[test]
fn codex_root_records_last_turn_output_on_stop() {
let tmp = tempfile::tempdir().unwrap();
let transcript = tmp.path().join("rollout.jsonl");
write_transcript(&transcript);
let tpath = transcript.to_str().unwrap();

let reg = Registry::default_agents();
let mut tr = reg.create("codex", "sess-outcome");
let ctx = SessionCtx {
session_id: "sess-outcome".into(),
config: None,
};

let mut ops = Vec::new();
ops.extend(
tr.handle(
&envelope("sess-outcome", "SessionStart", tpath, json!({ "source": "startup" })),
&ctx,
)
.unwrap(),
);
ops.extend(
tr.handle(&envelope("sess-outcome", "Stop", tpath, json!({})), &ctx)
.unwrap(),
);
ops.extend(tr.flush(&ctx).unwrap());

let rows = reduce(ops);
let turn = find(&rows, SpanType::Task, "turn: t1");
assert_eq!(turn.output, Some(json!("Here are the files.")));
let root = find(&rows, SpanType::Task, "codex: myapp");
assert_eq!(
root.output,
Some(json!("Here are the files.")),
"the session outcome must land on the root span"
);
}

#[test]
fn codex_root_preserves_canonical_source_across_lifecycle_events() {
let tmp = tempfile::tempdir().unwrap();
Expand Down
Loading