diff --git a/crates/flowproof-trace/schema/cassette-v1.schema.json b/crates/flowproof-trace/schema/cassette-v1.schema.json new file mode 100644 index 00000000..c50f3e0f --- /dev/null +++ b/crates/flowproof-trace/schema/cassette-v1.schema.json @@ -0,0 +1,127 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/automators-com/flowproof/blob/main/crates/flowproof-trace/schema/cassette-v1.schema.json", + "title": "flowproof agent-trace cassette lane v1", + "description": "The `cassette` lane an `app: agent` trace document carries: the recorded model-boundary trajectory, and the per-delivery metadata a multi-turn `conversation:` flow adds. Deliberately describes the lane only, not the whole agent-trace document.", + "$ref": "#/$defs/cassette", + "$defs": { + "cassette": { + "type": "object", + "description": "A recorded trajectory: every model call the system under test made, in order.", + "required": ["turns"], + "additionalProperties": false, + "properties": { + "turns": { + "type": "array", + "description": "The model-boundary exchanges, in the order they happened. Matched strictly by position at replay.", + "items": { "$ref": "#/$defs/turn" } + }, + "deliveries": { + "type": "array", + "description": "Per-delivery metadata for a multi-delivery `conversation:` flow, one entry per delivery in order. Absent for every single-delivery trace, including every trace recorded before this field existed.", + "items": { "$ref": "#/$defs/delivery_meta" } + } + } + }, + "turn": { + "type": "object", + "description": "One request/response exchange at the model boundary.", + "required": ["request", "response"], + "additionalProperties": false, + "properties": { + "protocol": { + "description": "The API dialect this exchange was recorded in. Absent when it equals the `openai` default, so every v1 trace round-trips byte-identical.", + "enum": ["openai", "anthropic"] + }, + "request": { "$ref": "#/$defs/turn_request" }, + "response": { "$ref": "#/$defs/turn_response" }, + "delivery_index": { + "type": "integer", + "minimum": 0, + "description": "Which delivery produced this turn. A delivery is one `conversation:` user message plus every turn it provoked before the agent settled - a coarser unit than a turn. Absent when zero, so every single-delivery trace round-trips byte-identical." + } + } + }, + "turn_request": { + "type": "object", + "description": "What the system under test sent to the model.", + "required": ["model", "messages"], + "additionalProperties": false, + "properties": { + "model": { "type": "string" }, + "messages": { + "type": "array", + "items": { "$ref": "#/$defs/message" } + }, + "tools": { + "type": "array", + "description": "Tool names offered, in the order the request listed them. Absent when empty.", + "items": { "type": "string" } + } + } + }, + "turn_response": { + "type": "object", + "description": "What the model answered.", + "required": ["message"], + "additionalProperties": false, + "properties": { + "message": { "$ref": "#/$defs/message" }, + "stop_reason": { + "type": "string", + "description": "The wire-level stop reason (`end_turn`, `tool_use`, ...). SERVED but NEVER MATCHED: it is an output of the turn, not part of the request identity." + } + } + }, + "message": { + "type": "object", + "description": "One message in a chat completion, in the shape the OpenAI-compatible wire format uses.", + "required": ["role"], + "additionalProperties": false, + "properties": { + "role": { "type": "string" }, + "content": { "type": "string" }, + "tool_calls": { + "type": "array", + "description": "Calls the model asked for. Assistant messages only; absent when empty.", + "items": { "$ref": "#/$defs/tool_call" } + }, + "tool_call_id": { + "type": "string", + "description": "Which call this message answers. Tool messages only." + } + } + }, + "tool_call": { + "type": "object", + "description": "A tool invocation the model asked for.", + "required": ["id", "name", "arguments"], + "additionalProperties": false, + "properties": { + "id": { "type": "string" }, + "name": { "type": "string" }, + "arguments": { + "type": "string", + "description": "Stays a STRING because that is what the wire carries: re-encoding it as JSON would silently reorder keys and lose the exact bytes an assertion may care about." + } + } + }, + "delivery_meta": { + "type": "object", + "description": "Metadata about one delivery in a multi-delivery `conversation:` flow. NOT matched against at replay - the wire never re-sends it - it exists so a cassette or `heal` diff reads as an actual transcript instead of a bare turn count.", + "required": ["user", "turn_count"], + "additionalProperties": false, + "properties": { + "user": { + "type": "string", + "description": "The user message as recorded, verbatim. Not redacted: it is the operator's own flow-file content, already visible in the `.flow.yaml`." + }, + "turn_count": { + "type": "integer", + "minimum": 0, + "description": "How many turns this delivery produced." + } + } + } + } +} diff --git a/crates/flowproof-trace/tests/cassette_conformance.rs b/crates/flowproof-trace/tests/cassette_conformance.rs new file mode 100644 index 00000000..86ed0ffb --- /dev/null +++ b/crates/flowproof-trace/tests/cassette_conformance.rs @@ -0,0 +1,170 @@ +//! Keeps the `Cassette` serde types and the cassette JSON Schema in +//! agreement. The schema exists because `delivery_index`/`deliveries` are +//! the on-disk shape of multi-turn `conversation:` flows, and a field that +//! only a Rust struct describes is undocumented in the form a consumer can +//! actually check against. +//! +//! The load-bearing property is the one the multi-turn work rests on: the +//! new fields are ABSENT at their defaults, so every cassette recorded +//! before they existed still validates. + +use flowproof_trace::cassette::{ + default_protocol, Cassette, DeliveryMeta, Message, ToolCall, Turn, TurnRequest, TurnResponse, +}; + +const SCHEMA: &str = include_str!("../schema/cassette-v1.schema.json"); + +fn validator() -> jsonschema::Validator { + let schema: serde_json::Value = serde_json::from_str(SCHEMA).expect("schema is valid JSON"); + jsonschema::validator_for(&schema).expect("schema compiles") +} + +fn turn(user: &str, reply: &str, delivery_index: usize) -> Turn { + Turn { + protocol: default_protocol(), + request: TurnRequest { + model: "gpt-4o".into(), + messages: vec![Message::new("user", user)], + tools: Vec::new(), + }, + response: TurnResponse { + message: Message::new("assistant", reply), + stop_reason: None, + }, + delivery_index, + } +} + +fn assert_validates(validator: &jsonschema::Validator, value: &serde_json::Value) { + assert!( + validator.validate(value).is_ok(), + "failed schema validation: {:?}", + validator.iter_errors(value).next() + ); +} + +/// A single-delivery cassette - the shape every pre-`conversation:` trace +/// has - validates, and carries neither new field. +#[test] +fn a_single_delivery_cassette_validates_without_the_conversation_fields() { + let validator = validator(); + let cassette = Cassette { + turns: vec![turn("What is the weather?", "Sunny.", 0)], + deliveries: Vec::new(), + }; + let json = serde_json::to_value(&cassette).expect("cassette serializes"); + assert_validates(&validator, &json); + + let text = serde_json::to_string(&cassette).expect("cassette serializes"); + assert!(!text.contains("delivery_index"), "{text}"); + assert!(!text.contains("deliveries"), "{text}"); +} + +/// A multi-delivery cassette validates with both fields present, and +/// round-trips through the typed model without drifting from the schema. +#[test] +fn a_multi_delivery_cassette_validates_and_round_trips() { + let validator = validator(); + let cassette = Cassette { + turns: vec![ + turn("Cancel order A-4471.", "Are you sure?", 0), + turn("Yes, go ahead.", "Cancelled.", 1), + ], + deliveries: vec![ + DeliveryMeta { + user: "Cancel order A-4471.".into(), + turn_count: 1, + }, + DeliveryMeta { + user: "Yes, go ahead.".into(), + turn_count: 1, + }, + ], + }; + let json = serde_json::to_value(&cassette).expect("cassette serializes"); + assert_validates(&validator, &json); + + let parsed: Cassette = serde_json::from_value(json.clone()).expect("cassette parses"); + assert_eq!(parsed, cassette, "round-trip reproduces the cassette"); + let reserialized = serde_json::to_value(&parsed).expect("cassette serializes"); + assert_eq!(reserialized, json, "round-trip reproduces the bytes"); + assert_validates(&validator, &reserialized); +} + +/// Tool calls and the Anthropic dialect are part of the recorded shape, so +/// the schema has to accept them rather than only the happy text path. +#[test] +fn a_tool_calling_anthropic_turn_validates() { + let validator = validator(); + let mut t = turn("Cancel it.", "", 1); + t.protocol = "anthropic".into(); + t.request.tools = vec!["cancel_order".into()]; + t.response.stop_reason = Some("tool_use".into()); + t.response.message = Message { + role: "assistant".into(), + content: None, + tool_calls: vec![ToolCall { + id: "call_1".into(), + name: "cancel_order".into(), + arguments: "{\"id\":\"A-4471\"}".into(), + }], + tool_call_id: None, + }; + let cassette = Cassette { + turns: vec![t], + deliveries: vec![DeliveryMeta { + user: "Cancel it.".into(), + turn_count: 1, + }], + }; + let json = serde_json::to_value(&cassette).expect("cassette serializes"); + assert_validates(&validator, &json); +} + +/// The schema is an instrument, not documentation: it has to REFUSE the +/// shapes that would mean a broken recording. +#[test] +fn the_schema_refuses_malformed_cassettes() { + let validator = validator(); + + // A turn missing its response is a half-recorded exchange. + let no_response = serde_json::json!({ + "turns": [{"request": {"model": "gpt-4o", "messages": []}}] + }); + assert!(validator.validate(&no_response).is_err()); + + // delivery_index counts deliveries; a negative one is meaningless. + let negative_index = serde_json::json!({ + "turns": [{ + "request": {"model": "gpt-4o", "messages": []}, + "response": {"message": {"role": "assistant"}}, + "delivery_index": -1 + }] + }); + assert!(validator.validate(&negative_index).is_err()); + + // A dialect nothing can replay must not validate. + let unknown_protocol = serde_json::json!({ + "turns": [{ + "protocol": "cohere", + "request": {"model": "gpt-4o", "messages": []}, + "response": {"message": {"role": "assistant"}} + }] + }); + assert!(validator.validate(&unknown_protocol).is_err()); + + // DeliveryMeta without its turn_count cannot describe a window. + let partial_delivery = serde_json::json!({ + "turns": [], + "deliveries": [{"user": "hi"}] + }); + assert!(validator.validate(&partial_delivery).is_err()); + + // An unknown key is a typo or a field this schema has not caught up + // with; either way it should be reported, not silently accepted. + let unknown_key = serde_json::json!({ + "turns": [], + "delivery": [] + }); + assert!(validator.validate(&unknown_key).is_err()); +} diff --git a/docs/trace-format.md b/docs/trace-format.md index c0c2f780..f8323e2f 100644 --- a/docs/trace-format.md +++ b/docs/trace-format.md @@ -50,6 +50,50 @@ these fields existed) is byte-identical: (Two further fields, `id` and `answer`, are reserved for the v3.4 server-initiated REQUEST slice and stay absent until then.) +### Cassette lane (`app: agent`) + +The `cassette` key is the recorded trajectory itself: `{"turns":[…]}`, one +entry per model-boundary exchange, matched strictly by position at replay. +Schema: +[`crates/flowproof-trace/schema/cassette-v1.schema.json`](../crates/flowproof-trace/schema/cassette-v1.schema.json). + +A multi-turn [`conversation:`](agent-testing/status-and-scope.md#multi-turn-conversations) +flow adds two ADDITIVE fields, both omitted at their defaults so every +single-delivery cassette - including every one recorded before these fields +existed - serializes byte-identical: + +```json +"cassette": { + "turns": [ + {"request": {…}, "response": {…}}, + {"request": {…}, "response": {…}, "delivery_index": 1} + ], + "deliveries": [ + {"user": "Cancel order A-4471.", "turn_count": 1}, + {"user": "Yes, go ahead.", "turn_count": 1} + ] +} +``` + +- `delivery_index` on a turn (omitted when `0`) is which **delivery** + produced it. A delivery is a coarser unit than a turn: one `conversation:` + user message plus every turn it provoked before the agent settled. The + windows **tile the turn list exactly**: each delivery's turns are + contiguous, each delivery has at least one turn, indexes are dense and + ascending, and no turn falls outside a window. A recording that violates + any of those is refused rather than written, which is what stops a timeout + or an agent that exited early from minting a passing cassette. +- `deliveries` (omitted when empty) is one entry per delivery in order, + each `{"user":…,"turn_count":…}`. The `user` text is stored verbatim and + is NOT redacted: it is the operator's own flow-file content, already + visible in the `.flow.yaml`. + +**`deliveries` is reporting, not authority.** It is never matched against at +replay - the wire never re-sends it - and exists so a cassette or `heal` +diff reads as an actual transcript instead of a bare turn count. What replay +enforces is the turns and their `delivery_index` grouping; editing a +`turn_count` changes what a diff *says*, not what a run *verdicts*. + ### Side-effect lane (`app: agent`) A run the seccomp observation mechanism ran for (Linux, `command:` driver,