Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion config/proof-pin.toml
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ max_output_tokens_ceiling = 8192
inference_offer_commitment_alg = "sha256"
eval_executor_schema_version = 1
gpu_class = "1x"
max_proof_deadline_s_ceiling = 7200
max_proof_deadline_s_ceiling = 14400
allowed_lium_template_prefixes = ["proof-eval-"]
eval_executor_commitment_alg = "sha256"
eval_image = "ghcr.io/cortexlm/proof-eval"
Expand Down
4 changes: 2 additions & 2 deletions crates/proof-executor/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -505,12 +505,12 @@ mod tests {
);
assert!(matches!(
over.validate(&p),
Err(ExecutorOfferError::BadDeadline(7_201, 7_200))
Err(ExecutorOfferError::BadDeadline(14_401, 14_400))
));
let zero = offer_for("proof-eval-78b614a1f51c", 0, &p);
assert!(matches!(
zero.validate(&p),
Err(ExecutorOfferError::BadDeadline(0, 7_200))
Err(ExecutorOfferError::BadDeadline(0, 14_400))
));
let mut tight = p.clone();
tight.max_proof_deadline_s_ceiling = 3_600;
Expand Down
10 changes: 5 additions & 5 deletions crates/proof-executor/src/plan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -323,30 +323,30 @@ mod tests {
// up to the ceiling — never past it.
let short_offer = offer_for("proof-eval-78b614a1f51c", 600, &p);
let longer = HarvestOverrides {
deadline_secs: Some(7_200),
deadline_secs: Some(14_400),
..HarvestOverrides::default()
};
assert_eq!(
executor_plan(&p, Some(&short_offer), &topic(), &longer)
.expect("plan")
.deadline_s,
7_200
14_400
);
let past = HarvestOverrides {
deadline_secs: Some(7_201),
deadline_secs: Some(14_401),
..HarvestOverrides::default()
};
assert!(matches!(
executor_plan(&p, Some(&offer()), &topic(), &past),
Err(ExecutorOfferError::BadDeadline(7_201, 7_200))
Err(ExecutorOfferError::BadDeadline(14_401, 14_400))
));
let zero = HarvestOverrides {
deadline_secs: Some(0),
..HarvestOverrides::default()
};
assert!(matches!(
executor_plan(&p, Some(&offer()), &topic(), &zero),
Err(ExecutorOfferError::BadDeadline(0, 7_200))
Err(ExecutorOfferError::BadDeadline(0, 14_400))
));
// A topic tighten still applies on top of the override.
let mut t = topic();
Expand Down
4 changes: 2 additions & 2 deletions crates/proof-harvest/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1847,10 +1847,10 @@ mod tests {
),
(
HarvestOverrides {
deadline_secs: Some(7_201),
deadline_secs: Some(14_401),
..HarvestOverrides::default()
},
"max_proof_deadline_s = 7201",
"max_proof_deadline_s = 14401",
),
(
HarvestOverrides {
Expand Down
4 changes: 2 additions & 2 deletions crates/proof-http/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2036,7 +2036,7 @@ mod tests {
// ceilings are public regardless.
assert!(body["eval_executor"].is_null(), "{body}");
assert_eq!(body["executor"]["gpu_class"], "1x");
assert_eq!(body["executor"]["max_proof_deadline_s_ceiling"], 7_200);
assert_eq!(body["executor"]["max_proof_deadline_s_ceiling"], 14_400);
}

#[tokio::test]
Expand Down Expand Up @@ -2077,7 +2077,7 @@ mod tests {
assert_eq!(view["ready"], true, "{view}");
assert!(view["reason"].is_null(), "{view}");
assert_eq!(view["eval_executor"]["offer_id"], "lium-1x-v0");
assert_eq!(view["pin"]["max_proof_deadline_s_ceiling"], 7_200);
assert_eq!(view["pin"]["max_proof_deadline_s_ceiling"], 14_400);
let dump = view.to_string();
assert!(!dump.contains("api_key"), "{dump}");
assert!(!dump.contains("/run/base"), "{dump}");
Expand Down
7 changes: 4 additions & 3 deletions crates/proof-task/src/executor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,9 @@ pub const EVAL_EXECUTOR_GPU_CLASS: &str = "1x";
pub const EVAL_EXECUTOR_GPU_COUNT: u32 = 1;

/// Pin `max_proof_deadline_s_ceiling`: longest proof deadline an offer or a
/// topic may declare (two hours).
pub const MAX_PROOF_DEADLINE_S_CEILING: u64 = 7_200;
/// topic may declare (four hours). A pin may only tighten it below this
/// lock, and the live offer may be shorter still.
pub const MAX_PROOF_DEADLINE_S_CEILING: u64 = 14_400;

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Align submission timeouts

This change permits Proof deadlines up to 14,400 seconds, but synchronous submission still has a 7,200-second gateway deadline and the default client uses the same limit. A valid proof that finishes after two hours returns a gateway or client timeout instead of the expected 201 Created, even though its detached scoring work can finish and persist afterward. Raise those submission budgets to cover the supported deadline range, including any transport margin, or change the submission contract to acknowledge work before scoring completes.

Knowledge Base Used: Proof tasks, challenge service, and scoring

Artifacts

Command output from the check

Command output from the check

  • Executed the authored paused-time gateway contract test against the PR state; it reports HTTP 504 Gateway Timeout with `upstream deadline` while a 7,201-second detached score persists, confirming the mismatch.

Evidence from the check

  • The authored Rust test invokes the real gateway detached Proof timeout function and verifies its 504 response and post-timeout score persistence.

Evidence from the check

  • The authored shell command temporarily installs the narrow test with Tokio test time enabled, runs it, and restores repository files afterward.

Command output from the check

  • Executed the current executor plan boundary test, which includes acceptance of a 14,400-second override, confirming the changed admission side.

View artifacts

T-Rex Ran code and verified through T-Rex


/// Pin `eval_executor_commitment_alg`.
pub const EVAL_EXECUTOR_COMMITMENT_ALG: &str = "sha256";
Expand Down Expand Up @@ -88,7 +89,7 @@ mod tests {
assert_eq!(EVAL_EXECUTOR_SCHEMA_VERSION, 1);
assert_eq!(EVAL_EXECUTOR_GPU_CLASS, "1x");
assert_eq!(EVAL_EXECUTOR_GPU_COUNT, 1);
assert_eq!(MAX_PROOF_DEADLINE_S_CEILING, 7_200);
assert_eq!(MAX_PROOF_DEADLINE_S_CEILING, 14_400);
assert_eq!(EVAL_EXECUTOR_COMMITMENT_ALG, "sha256");
}

Expand Down
7 changes: 5 additions & 2 deletions crates/proof-task/src/pin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -508,7 +508,7 @@ max_output_tokens_ceiling = 8192
inference_offer_commitment_alg = "sha256"
eval_executor_schema_version = 1
gpu_class = "1x"
max_proof_deadline_s_ceiling = 7200
max_proof_deadline_s_ceiling = 14400
allowed_lium_template_prefixes = ["proof-eval-"]
eval_executor_commitment_alg = "sha256"
eval_image = "{EVAL_IMAGE}"
Expand All @@ -530,7 +530,7 @@ stratum_size = 24
assert!(p.proxy_model.is_empty());
assert!(p.proxy_models.is_empty());
assert_eq!(p.gpu_class, "1x");
assert_eq!(p.max_proof_deadline_s_ceiling, 7_200);
assert_eq!(p.max_proof_deadline_s_ceiling, 14_400);
assert_eq!(p.allowed_lium_template_prefixes, vec!["proof-eval-"]);
assert!(p.allows_template("proof-eval-78b614a1f51c"));
assert!(!p.allows_template("prism-recipe-v10"));
Expand Down Expand Up @@ -597,6 +597,9 @@ topic_pubkey = "{}"
#[test]
fn deadline_ceiling_may_tighten_never_loosen() {
let mut p = pin();
p.max_proof_deadline_s_ceiling = MAX_PROOF_DEADLINE_S_CEILING;
p.validate().expect("the crate lock is legal");
assert_eq!(MAX_PROOF_DEADLINE_S_CEILING, 14_400);
p.max_proof_deadline_s_ceiling = 3_600;
p.validate().expect("tighter ceiling is legal");
p.max_proof_deadline_s_ceiling = MAX_PROOF_DEADLINE_S_CEILING + 1;
Expand Down
4 changes: 2 additions & 2 deletions crates/proof-task/tests/committed_pin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ fn committed_pin_is_proof_with_a_real_eval_digest() {
}

/// The executor ceilings are the 2026-09-08 lock: schema 1, `1x` only,
/// two-hour deadline ceiling, sha256 commitment, and a `proof-eval-` template
/// four-hour deadline ceiling, sha256 commitment, and a `proof-eval-` template
/// allowlist that matches the digest-scoped harvest template name.
#[test]
fn committed_pin_locks_the_one_gpu_executor_ceilings() {
Expand All @@ -117,7 +117,7 @@ fn committed_pin_locks_the_one_gpu_executor_ceilings() {
assert_eq!(p.gpu_class, EVAL_EXECUTOR_GPU_CLASS);
assert_eq!(p.gpu_class, "1x");
assert_eq!(p.max_proof_deadline_s_ceiling, MAX_PROOF_DEADLINE_S_CEILING);
assert_eq!(p.max_proof_deadline_s_ceiling, 7_200);
assert_eq!(p.max_proof_deadline_s_ceiling, 14_400);
assert_eq!(p.eval_executor_commitment_alg, EVAL_EXECUTOR_COMMITMENT_ALG);
assert_eq!(p.allowed_lium_template_prefixes, vec!["proof-eval-"]);
let digest_hex = p.eval_image_digest.trim_start_matches("sha256:");
Expand Down
4 changes: 2 additions & 2 deletions deploy/env/proof-challenge.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,8 @@ PROOF_SIM_STUB_WIN=false

# Fallback seconds the eval image gets when no executor deadline was resolved.
# The resolved EvalExecutorOffer max_proof_deadline_s IS the pod timeout and
# is never clamped by this value. Default = pin ceiling (7200).
# PROOF_EVAL_TIMEOUT_SECS=7200
# is never clamped by this value. Default = pin ceiling (14400).
# PROOF_EVAL_TIMEOUT_SECS=14400

# Live RLM judge InferenceOffer (operator state, never git). The eval image
# calls this backend to score miner submissions. Miners do not bind it.
Expand Down
2 changes: 1 addition & 1 deletion docs/COMPLETENESS.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ specs (`DESIGN_CHALLENGE.md`, `PRISM.md`) remain for `xtask` gates. Leftover
| Compose / images | **done** | Default compose + `images.yml` target `proof-challenge`. |
| Eval pin | **done** | `config/proof-pin.toml` — `eval_image` `ghcr.io/cortexlm/proof-eval`, digest `sha256:78b614a1…` (publish-proof-eval-image run 33892650063, commit `51f937c7`). No HF bake; `proxy_model` stays empty. Live submits still **503** until harvest is wired, a baseline is sealed, and ≥1 topic is open. Do not re-pin a guessed sha256. |
| Inference offer | **v0** | Digest-pinned RLM **judge** backend (`proof-eval` / harvest call it). Pin `[inference]` defaults plus schema v1 / ceilings / modes / commitment. `config_commitment` hashes config knobs **and** `provider.base_url`; a topic that spoofs origin is **503** before lattice. Topic `require_judge_offer_commitment` is optional and not a miner bind. Live `InferenceOffer` is operator state. Auth is `PROOF_INFERENCE_API_KEY_FILE` staged as harvest `teacher.env` (never git, never `/v1/status`). Missing/closed/judge down / missing key → `can_score=false` / 503. No baked Qwen; architecture ≠ HF stays retired. |
| Eval executor offer | **v0** | `crates/proof-executor`: live `1x` `EvalExecutorOffer` (Lium template, `machine_shape`, `max_proof_deadline_s`, digest, `config_commitment`, status) — a sibling of the judge offer, not the same document. Pin ceilings `eval_executor_schema_version` / `gpu_class = "1x"` / `max_proof_deadline_s_ceiling = 7200` / optional `allowed_lium_template_prefixes` / `eval_executor_commitment_alg`. Public on `GET /v1/status` + `GET /v1/proof/executor`; rotated via `POST /v1/admin/proof/executor` (in-memory until restart; boot from `PROOF_EVAL_EXECUTOR_OFFER_FILE`). Topic tighten-only `eval_executor.{require_offer_commitment, max_proof_deadline_s}`, no per-topic `machine_id`. Lium path: missing/closed/shape ≠ `1x` → `can_score=false` / 503; harvest rents the offer's digest-scoped template (raw Lium UUIDs refused under any allowlist; the resolver binds the template to `eval_image@digest`) at exactly `1x` (`rent_gpu_count ≠ 1` aborts pre-rent) and holds the run to the deadline (the deadline is the pod `timeout`, never clamped by the host fallback; harvest wait = deadline + grace; wrapper-cut run → 503 + `stdout_tail`, external SIGKILL named separately). `PROOF_HARVEST_TEMPLATE_ID` / `_GPU_COUNT` / `_DEADLINE_SECS` hot-swap under the pin ceilings; refused when the topic pins the offer commitment; the run request and row stamp the commitment of what actually ran. Sim does not consult it. No live Lium rent in CI. |
| Eval executor offer | **v0** | `crates/proof-executor`: live `1x` `EvalExecutorOffer` (Lium template, `machine_shape`, `max_proof_deadline_s`, digest, `config_commitment`, status) — a sibling of the judge offer, not the same document. Pin ceilings `eval_executor_schema_version` / `gpu_class = "1x"` / `max_proof_deadline_s_ceiling = 14400` / optional `allowed_lium_template_prefixes` / `eval_executor_commitment_alg`. Public on `GET /v1/status` + `GET /v1/proof/executor`; rotated via `POST /v1/admin/proof/executor` (in-memory until restart; boot from `PROOF_EVAL_EXECUTOR_OFFER_FILE`). Topic tighten-only `eval_executor.{require_offer_commitment, max_proof_deadline_s}`, no per-topic `machine_id`. Lium path: missing/closed/shape ≠ `1x` → `can_score=false` / 503; harvest rents the offer's digest-scoped template (raw Lium UUIDs refused under any allowlist; the resolver binds the template to `eval_image@digest`) at exactly `1x` (`rent_gpu_count ≠ 1` aborts pre-rent) and holds the run to the deadline (the deadline is the pod `timeout`, never clamped by the host fallback; harvest wait = deadline + grace; wrapper-cut run → 503 + `stdout_tail`, external SIGKILL named separately). `PROOF_HARVEST_TEMPLATE_ID` / `_GPU_COUNT` / `_DEADLINE_SECS` hot-swap under the pin ceilings; refused when the topic pins the offer commitment; the run request and row stamp the commitment of what actually ran. Sim does not consult it. No live Lium rent in CI. |
| Topics | **done** | sr25519 under the `proof` trust-root key (`base-proof-topic-v1`). Admin `POST /v1/admin/proof/topics`. A topic must be sealed to `open`. |
| Topic installs (dynamic topics P0) | **skeleton** | `bins/proof-admin` wraps the **existing** publish path rather than adding a registry: `topic validate` runs the same acceptance `POST /v1/admin/proof/topics` runs (`TopicDocument::validate` + `verify_signature`, against `config/proof-pin.toml`), `topic install --dry-run` prints that publish call plus the host env, and `topic list` / `topic show` are a read-only view of the existing `proof_topic_version` rows (`RlmStore::latest_topics`, migration `0020`). Bundle schema v1 (`crates/proof-topic-bundle`) carries the signed document verbatim plus a `host` block that must **agree** with it (a contradiction is a reject); runner/pack/custom-id bindings are the document's own `constraints.params`. **Schema: `0024_proof_topic_alias.sql` only** — it adds `proof_topic_alias` plus a `BEFORE INSERT`/`UPDATE` trigger pair that fails closed when an alias would shadow a published slug (**publish-path integrity, not scoring math**); it does not `ALTER` or `DROP` anything, and `0020` tables keep their columns, keys, and grants. **No route change, no scoring change** — a real install is not implemented (exit 3), and `topic enable` / `disable` / `seal` are stubs (a topic's lifecycle is the document's `status`). No route change (P1), allocator change (P2), full install (P3), or removal of the compiled-in topic bindings (P4). Locked defaults: first topic slug **`tb4`** with temporary alias **`tbench`** (`proof_topic_alias`, migration `0024` — a row carries only `alias → topic_id`, so it cannot drift from the topic), shared challenge DB with a `topic_id` discriminant, and **metal `--env metal` is Owner-only behind `--owner-metal-ack` with staging first** (staging is never gated). `tbench` is both the alias and the runner registry's custom id; the alias is temporary, the custom id is the scoring binding. **Topics are RLM-owned:** the bundle's `rlm` section (rules / migrations / apis / submission_format / scoring) is handed to the RLM verbatim and Rust never interprets it; two guard tests fail the build if a topic id or a topic-specific rule/metric/format appears in the bundle crate's or the CLI's logic. |
| Holdout | **done** | Per-topic operator file (`PROOF_HOLDOUT_FILE`). Commitment in the topic document, never in the pin. `xtask proof-holdout --topic-id`. |
Expand Down
4 changes: 2 additions & 2 deletions docs/PROOF.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ baseline + an open topic are on the host.
- **Where** the image runs is the live `EvalExecutorOffer` — a sibling of
the judge `InferenceOffer`, never the same document. The pin carries only
ceilings: `eval_executor_schema_version = 1`, `gpu_class = "1x"`,
`max_proof_deadline_s_ceiling = 7200`, optional
`max_proof_deadline_s_ceiling = 14400`, optional
`allowed_lium_template_prefixes` (`["proof-eval-"]`, the digest-scoped
harvest template name `proof-eval-<12 hex>`), and
`eval_executor_commitment_alg = sha256`. The live offer
Expand Down Expand Up @@ -828,7 +828,7 @@ id shapes shared with `proof-task`).
| `constraints.params.experiment_vcpus` / `experiment_mem_mib` / `experiment_disk_mib` | The topic's size ask: silent = the operator defaults (lock 16 vCPU / 32 GiB / 32 GiB disk), an ask may go up to the ceilings (lock 16 vCPU / 32 GiB — the default is the ceiling; disk ≥ 16 GiB); over = 503, never a clamp |
| `checklist` | ≤64 `{id, text}` anti-cheat rules (unique slug ids), version 1 of the rule set |
| `eval_executor.require_offer_commitment` | 64-hex pin against the live `1x` `EvalExecutorOffer` (`proof-executor`) |
| `eval_executor.max_proof_deadline_s` | Tighten-only against pin `max_proof_deadline_s_ceiling` (7200 s; the live offer may be shorter) |
| `eval_executor.max_proof_deadline_s` | Tighten-only against pin `max_proof_deadline_s_ceiling` (14400 s; the live offer may be shorter) |

### Miner BYOK (`env` on the submit body)

Expand Down
2 changes: 1 addition & 1 deletion docs/external-miner/proof.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ finishes installing its scoring path — nothing is evaluated or rented yet.
|--------------|----------------|
| `eval_image_digest` | Must be a `sha256:…` pin (live pin is `sha256:78b614a1…`). Empty → **503** |
| `inference_offer` | Public RLM **judge** backend (id, kind, mode, model_ref, token caps, commitment, status). Missing/closed/misconfigured → **503**. You do not pass an offer id |
| `eval_executor` | Public `1x` **executor**: the Lium machine class your recipe is re-run on (`lium_template_id`, `machine_shape`, `max_proof_deadline_s`, commitment, status). Your recipe must finish inside `max_proof_deadline_s` (≤ pin ceiling 7200 s; a topic may name a shorter one) on **one** GPU — the host never rents more. Missing/closed/any shape but `1x` → **503**. You do not pass or rent it |
| `eval_executor` | Public `1x` **executor**: the Lium machine class your recipe is re-run on (`lium_template_id`, `machine_shape`, `max_proof_deadline_s`, commitment, status). Your recipe must finish inside `max_proof_deadline_s` (≤ pin ceiling 14400 s; a topic may name a shorter one) on **one** GPU — the host never rents more. Missing/closed/any shape but `1x` → **503**. You do not pass or rent it |
| `open_topics` empty | No currently `open` signed topic with a sealed baseline → **503** |
| `scorable_topics` | Open topics whose family's scorer is wired on this host and that are scored right now. An open topic **not** listed here (a `custom` topic whose runner is not registered or not wired; an `nll` / `throughput` topic on a host whose Lium harvest is not wired) answers **503** — unless it is in `deferred_topics` |
| `deferred_topics` | Open topics whose signed document sets `constraints.params.defer_scoring = "true"`: the operator is still installing the baseline / harness. Submits are accepted (**201**) and stored as **`queued`**; nothing is evaluated or rented until the operator lifts the flag and drains the queue, oldest first |
Expand Down
2 changes: 1 addition & 1 deletion docs/runbooks/proof-submit-e2e.md
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ curl -sS "$BASE/v1/status"
# "open_topics": ["dt-no-ib-v0", "muon-vs-adamw-10m-v0"],
# "inference_offer": { "offer_id": "openrouter-glm53flash-v0", "status": "open", ... },
# "eval_executor": null, # sim rents nothing; Lium needs an open 1x offer
# "executor": { "gpu_class": "1x", "max_proof_deadline_s_ceiling": 7200, ... }
# "executor": { "gpu_class": "1x", "max_proof_deadline_s_ceiling": 14400, ... }
# }
# Never contains api_key, base_url, or holdout records.

Expand Down
2 changes: 1 addition & 1 deletion xtask/src/proof_executor_offer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ mod tests {
let p = pin();
let err = build(&p, &args("8x", 7_200)).expect_err("8x");
assert!(err.contains("machine_shape"), "{err}");
let err = build(&p, &args("1x", 7_201)).expect_err("over ceiling");
let err = build(&p, &args("1x", 14_401)).expect_err("over ceiling");
assert!(err.contains("max_proof_deadline_s"), "{err}");
let mut unbound = args("1x", 600);
unbound.template_id = Some("prism-recipe-v10".into());
Expand Down