feat(trace): give the cassette lane a schema, including the conversation fields

crates/flowproof-trace/schema/cassette-v1.schema.json

@@ -0,0 +1,127 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/automators-com/flowproof/blob/main/crates/flowproof-trace/schema/cassette-v1.schema.json",
+  "title": "flowproof agent-trace cassette lane v1",
+  "description": "The `cassette` lane an `app: agent` trace document carries: the recorded model-boundary trajectory, and the per-delivery metadata a multi-turn `conversation:` flow adds. Deliberately describes the lane only, not the whole agent-trace document.",
+  "$ref": "#/$defs/cassette",
+  "$defs": {
+    "cassette": {
+      "type": "object",
+      "description": "A recorded trajectory: every model call the system under test made, in order.",
+      "required": ["turns"],
+      "additionalProperties": false,
+      "properties": {
+        "turns": {
+          "type": "array",
+          "description": "The model-boundary exchanges, in the order they happened. Matched strictly by position at replay.",
+          "items": { "$ref": "#/$defs/turn" }
+        },
+        "deliveries": {
+          "type": "array",
+          "description": "Per-delivery metadata for a multi-delivery `conversation:` flow, one entry per delivery in order. Absent for every single-delivery trace, including every trace recorded before this field existed.",
+          "items": { "$ref": "#/$defs/delivery_meta" }
+        }
+      }
+    },
+    "turn": {
+      "type": "object",
+      "description": "One request/response exchange at the model boundary.",
+      "required": ["request", "response"],
+      "additionalProperties": false,
+      "properties": {
+        "protocol": {
+          "description": "The API dialect this exchange was recorded in. Absent when it equals the `openai` default, so every v1 trace round-trips byte-identical.",
+          "enum": ["openai", "anthropic"]
+        },
+        "request": { "$ref": "#/$defs/turn_request" },
+        "response": { "$ref": "#/$defs/turn_response" },
+        "delivery_index": {
+          "type": "integer",
+          "minimum": 0,
+          "description": "Which delivery produced this turn. A delivery is one `conversation:` user message plus every turn it provoked before the agent settled - a coarser unit than a turn. Absent when zero, so every single-delivery trace round-trips byte-identical."
+        }
+      }
+    },
+    "turn_request": {
+      "type": "object",
+      "description": "What the system under test sent to the model.",
+      "required": ["model", "messages"],
+      "additionalProperties": false,
+      "properties": {
+        "model": { "type": "string" },
+        "messages": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/message" }
+        },
+        "tools": {
+          "type": "array",
+          "description": "Tool names offered, in the order the request listed them. Absent when empty.",
+          "items": { "type": "string" }
+        }
+      }
+    },
+    "turn_response": {
+      "type": "object",
+      "description": "What the model answered.",
+      "required": ["message"],
+      "additionalProperties": false,
+      "properties": {
+        "message": { "$ref": "#/$defs/message" },
+        "stop_reason": {
+          "type": "string",
+          "description": "The wire-level stop reason (`end_turn`, `tool_use`, ...). SERVED but NEVER MATCHED: it is an output of the turn, not part of the request identity."
+        }
+      }
+    },
+    "message": {
+      "type": "object",
+      "description": "One message in a chat completion, in the shape the OpenAI-compatible wire format uses.",
+      "required": ["role"],
+      "additionalProperties": false,
+      "properties": {
+        "role": { "type": "string" },
+        "content": { "type": "string" },
+        "tool_calls": {
+          "type": "array",
+          "description": "Calls the model asked for. Assistant messages only; absent when empty.",
+          "items": { "$ref": "#/$defs/tool_call" }
+        },
+        "tool_call_id": {
+          "type": "string",
+          "description": "Which call this message answers. Tool messages only."
+        }
+      }
+    },
+    "tool_call": {
+      "type": "object",
+      "description": "A tool invocation the model asked for.",
+      "required": ["id", "name", "arguments"],
+      "additionalProperties": false,
+      "properties": {
+        "id": { "type": "string" },
+        "name": { "type": "string" },
+        "arguments": {
+          "type": "string",
+          "description": "Stays a STRING because that is what the wire carries: re-encoding it as JSON would silently reorder keys and lose the exact bytes an assertion may care about."
+        }
+      }
+    },
+    "delivery_meta": {
+      "type": "object",
+      "description": "Metadata about one delivery in a multi-delivery `conversation:` flow. NOT matched against at replay - the wire never re-sends it - it exists so a cassette or `heal` diff reads as an actual transcript instead of a bare turn count.",
+      "required": ["user", "turn_count"],
+      "additionalProperties": false,
+      "properties": {
+        "user": {
+          "type": "string",
+          "description": "The user message as recorded, verbatim. Not redacted: it is the operator's own flow-file content, already visible in the `.flow.yaml`."
+        },
+        "turn_count": {
+          "type": "integer",
+          "minimum": 0,
+          "description": "How many turns this delivery produced."
+        }
+      }
+    }
+  }
+}

crates/flowproof-trace/tests/cassette_conformance.rs

@@ -0,0 +1,170 @@
+//! Keeps the `Cassette` serde types and the cassette JSON Schema in
+//! agreement. The schema exists because `delivery_index`/`deliveries` are
+//! the on-disk shape of multi-turn `conversation:` flows, and a field that
+//! only a Rust struct describes is undocumented in the form a consumer can
+//! actually check against.
+//!
+//! The load-bearing property is the one the multi-turn work rests on: the
+//! new fields are ABSENT at their defaults, so every cassette recorded
+//! before they existed still validates.
+
+use flowproof_trace::cassette::{
+    default_protocol, Cassette, DeliveryMeta, Message, ToolCall, Turn, TurnRequest, TurnResponse,
+};
+
+const SCHEMA: &str = include_str!("../schema/cassette-v1.schema.json");
+
+fn validator() -> jsonschema::Validator {
+    let schema: serde_json::Value = serde_json::from_str(SCHEMA).expect("schema is valid JSON");
+    jsonschema::validator_for(&schema).expect("schema compiles")
+}
+
+fn turn(user: &str, reply: &str, delivery_index: usize) -> Turn {
+    Turn {
+        protocol: default_protocol(),
+        request: TurnRequest {
+            model: "gpt-4o".into(),
+            messages: vec![Message::new("user", user)],
+            tools: Vec::new(),
+        },
+        response: TurnResponse {
+            message: Message::new("assistant", reply),
+            stop_reason: None,
+        },
+        delivery_index,
+    }
+}
+
+fn assert_validates(validator: &jsonschema::Validator, value: &serde_json::Value) {
+    assert!(
+        validator.validate(value).is_ok(),
+        "failed schema validation: {:?}",
+        validator.iter_errors(value).next()
+    );
+}
+
+/// A single-delivery cassette - the shape every pre-`conversation:` trace
+/// has - validates, and carries neither new field.
+#[test]
+fn a_single_delivery_cassette_validates_without_the_conversation_fields() {
+    let validator = validator();
+    let cassette = Cassette {
+        turns: vec![turn("What is the weather?", "Sunny.", 0)],
+        deliveries: Vec::new(),
+    };
+    let json = serde_json::to_value(&cassette).expect("cassette serializes");
+    assert_validates(&validator, &json);
+
+    let text = serde_json::to_string(&cassette).expect("cassette serializes");
+    assert!(!text.contains("delivery_index"), "{text}");
+    assert!(!text.contains("deliveries"), "{text}");
+}
+
+/// A multi-delivery cassette validates with both fields present, and
+/// round-trips through the typed model without drifting from the schema.
+#[test]
+fn a_multi_delivery_cassette_validates_and_round_trips() {
+    let validator = validator();
+    let cassette = Cassette {
+        turns: vec![
+            turn("Cancel order A-4471.", "Are you sure?", 0),
+            turn("Yes, go ahead.", "Cancelled.", 1),
+        ],
+        deliveries: vec![
+            DeliveryMeta {
+                user: "Cancel order A-4471.".into(),
+                turn_count: 1,
+            },
+            DeliveryMeta {
+                user: "Yes, go ahead.".into(),
+                turn_count: 1,
+            },
+        ],
+    };
+    let json = serde_json::to_value(&cassette).expect("cassette serializes");
+    assert_validates(&validator, &json);
+
+    let parsed: Cassette = serde_json::from_value(json.clone()).expect("cassette parses");
+    assert_eq!(parsed, cassette, "round-trip reproduces the cassette");
+    let reserialized = serde_json::to_value(&parsed).expect("cassette serializes");
+    assert_eq!(reserialized, json, "round-trip reproduces the bytes");
+    assert_validates(&validator, &reserialized);
+}
+
+/// Tool calls and the Anthropic dialect are part of the recorded shape, so
+/// the schema has to accept them rather than only the happy text path.
+#[test]
+fn a_tool_calling_anthropic_turn_validates() {
+    let validator = validator();
+    let mut t = turn("Cancel it.", "", 1);
+    t.protocol = "anthropic".into();
+    t.request.tools = vec!["cancel_order".into()];
+    t.response.stop_reason = Some("tool_use".into());
+    t.response.message = Message {
+        role: "assistant".into(),
+        content: None,
+        tool_calls: vec![ToolCall {
+            id: "call_1".into(),
+            name: "cancel_order".into(),
+            arguments: "{\"id\":\"A-4471\"}".into(),
+        }],
+        tool_call_id: None,
+    };
+    let cassette = Cassette {
+        turns: vec![t],
+        deliveries: vec![DeliveryMeta {
+            user: "Cancel it.".into(),
+            turn_count: 1,
+        }],
+    };
+    let json = serde_json::to_value(&cassette).expect("cassette serializes");
+    assert_validates(&validator, &json);
+}
+
+/// The schema is an instrument, not documentation: it has to REFUSE the
+/// shapes that would mean a broken recording.
+#[test]
+fn the_schema_refuses_malformed_cassettes() {
+    let validator = validator();
+
+    // A turn missing its response is a half-recorded exchange.
+    let no_response = serde_json::json!({
+        "turns": [{"request": {"model": "gpt-4o", "messages": []}}]
+    });
+    assert!(validator.validate(&no_response).is_err());
+
+    // delivery_index counts deliveries; a negative one is meaningless.
+    let negative_index = serde_json::json!({
+        "turns": [{
+            "request": {"model": "gpt-4o", "messages": []},
+            "response": {"message": {"role": "assistant"}},
+            "delivery_index": -1
+        }]
+    });
+    assert!(validator.validate(&negative_index).is_err());
+
+    // A dialect nothing can replay must not validate.
+    let unknown_protocol = serde_json::json!({
+        "turns": [{
+            "protocol": "cohere",
+            "request": {"model": "gpt-4o", "messages": []},
+            "response": {"message": {"role": "assistant"}}
+        }]
+    });
+    assert!(validator.validate(&unknown_protocol).is_err());
+
+    // DeliveryMeta without its turn_count cannot describe a window.
+    let partial_delivery = serde_json::json!({
+        "turns": [],
+        "deliveries": [{"user": "hi"}]
+    });
+    assert!(validator.validate(&partial_delivery).is_err());
+
+    // An unknown key is a typo or a field this schema has not caught up
+    // with; either way it should be reported, not silently accepted.
+    let unknown_key = serde_json::json!({
+        "turns": [],
+        "delivery": []
+    });
+    assert!(validator.validate(&unknown_key).is_err());
+}

docs/trace-format.md

@@ -50,6 +50,50 @@ these fields existed) is byte-identical:
   (Two further fields, `id` and `answer`, are reserved for the v3.4
   server-initiated REQUEST slice and stay absent until then.)
 
+### Cassette lane (`app: agent`)
+
+The `cassette` key is the recorded trajectory itself: `{"turns":[…]}`, one
+entry per model-boundary exchange, matched strictly by position at replay.
+Schema:
+[`crates/flowproof-trace/schema/cassette-v1.schema.json`](../crates/flowproof-trace/schema/cassette-v1.schema.json).
+
+A multi-turn [`conversation:`](agent-testing/status-and-scope.md#multi-turn-conversations)
+flow adds two ADDITIVE fields, both omitted at their defaults so every
+single-delivery cassette - including every one recorded before these fields
+existed - serializes byte-identical:
+
+```json
+"cassette": {
+  "turns": [
+    {"request": {…}, "response": {…}},
+    {"request": {…}, "response": {…}, "delivery_index": 1}
+  ],
+  "deliveries": [
+    {"user": "Cancel order A-4471.", "turn_count": 1},
+    {"user": "Yes, go ahead.", "turn_count": 1}
+  ]
+}
+```
+
+- `delivery_index` on a turn (omitted when `0`) is which **delivery**
+  produced it. A delivery is a coarser unit than a turn: one `conversation:`
+  user message plus every turn it provoked before the agent settled. The
+  windows **tile the turn list exactly**: each delivery's turns are
+  contiguous, each delivery has at least one turn, indexes are dense and
+  ascending, and no turn falls outside a window. A recording that violates
+  any of those is refused rather than written, which is what stops a timeout
+  or an agent that exited early from minting a passing cassette.
+- `deliveries` (omitted when empty) is one entry per delivery in order,
+  each `{"user":…,"turn_count":…}`. The `user` text is stored verbatim and
+  is NOT redacted: it is the operator's own flow-file content, already
+  visible in the `.flow.yaml`.
+
+**`deliveries` is reporting, not authority.** It is never matched against at
+replay - the wire never re-sends it - and exists so a cassette or `heal`
+diff reads as an actual transcript instead of a bare turn count. What replay
+enforces is the turns and their `delivery_index` grouping; editing a
+`turn_count` changes what a diff *says*, not what a run *verdicts*.
+
 ### Side-effect lane (`app: agent`)
 
 A run the seccomp observation mechanism ran for (Linux, `command:` driver,