Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,17 @@ cache-write input, and output are priced separately; requests above the
published 272,000-input-token threshold use the corresponding long-context
rates where OpenAI publishes them. Unknown models or missing price classes fail
closed as `UNPRICED MODEL MIX` rather than being guessed or treated as free.
Core, Extended, and DigBench trials use ephemeral threads that intentionally do
not appear in normal persisted session telemetry. While a subscription-funded
benchmark suite is active, Quota views replace the numeric API-equivalent
estimate with `SUBSCRIPTION BENCHMARK ACTIVE` instead of presenting partial
accounting. When each trial finishes, its authoritative benchmark usage is
folded into the same process-local accounting used for quota learning; missing
or unpriceable benchmark usage fails closed and restarts the learning anchor.
Benchmarks funded by `CODEXOMETER_BENCHMARK_API_KEY` are excluded because they
do not consume the displayed subscription quota. Benchmark threads remain
hidden from Monitor; hiding presentation does not exclude their aggregate
subscription impact from Quota views.
The embedded rates come from the
[official OpenAI API pricing page](https://developers.openai.com/api/docs/pricing)
and were retrieved on **2026-08-23**.
Expand Down
27 changes: 27 additions & 0 deletions internal/codex/benchmark.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,26 @@ const (
BenchmarkUsageRawResponses BenchmarkUsageSource = "raw-responses"
)

// BenchmarkBillingSource identifies whether benchmark model calls consume the
// Codex subscription quota displayed by Codexometer or a separately billed API
// key. Quota accounting must not infer this from identical token telemetry.
type BenchmarkBillingSource string

const (
BenchmarkBillingUnknown BenchmarkBillingSource = "unknown"
BenchmarkBillingSubscription BenchmarkBillingSource = "subscription"
BenchmarkBillingAPIKey BenchmarkBillingSource = "api-key"
)

// BenchmarkBillingSource reports the billing path selected for benchmark
// app-server sessions created by this client.
func (c Client) BenchmarkBillingSource() BenchmarkBillingSource {
if strings.TrimSpace(c.BenchmarkAPIKey) != "" {
return BenchmarkBillingAPIKey
}
return BenchmarkBillingSubscription
}

// BenchmarkResponseUsage is the exact upstream usage for one Responses API
// completion when the current Codex app-server exposes experimental raw events.
type BenchmarkResponseUsage struct {
Expand Down Expand Up @@ -1643,6 +1663,13 @@ func EstimateStandardAPIEqCost(model string, usage BenchmarkUsage) (float64, boo
return estimateSingleResponseAPICostWithIssue(model, usage)
}

// EstimateStandardAPIEqAggregateCost prices aggregate usage only when its
// standard API cost can be reconstructed without response boundaries. Long
// context usage therefore fails closed because its surcharge is per response.
func EstimateStandardAPIEqAggregateCost(model string, usage BenchmarkUsage) (float64, bool, string) {
return estimateAPICostWithIssue(model, usage)
}

func priceForModel(model string) (apiPrice, bool) {
model = strings.ToLower(strings.TrimSpace(model))
if price, ok := standardAPIPrices[model]; ok {
Expand Down
161 changes: 161 additions & 0 deletions internal/ui/benchmark_quota.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
package ui

import (
"fmt"
"math"
"strings"

"github.com/merefield/codexometer/internal/codex"
)

type benchmarkBillingProvider interface {
BenchmarkBillingSource() codex.BenchmarkBillingSource
}

// benchmarkQuotaAccounting retains only content-free, process-local totals for
// ephemeral benchmark calls. These calls never enter the persisted rollout
// reader, but subscription-funded calls do move the account quota windows.
type benchmarkQuotaAccounting struct {
billingSource codex.BenchmarkBillingSource
revision uint64
runSequence uint64
active bool
settling bool
costUSD float64
pricedCalls int64
unpricedCalls int64
accounted map[string]struct{}
}

func benchmarkQuotaAccountingFor(runner BenchmarkRunner) benchmarkQuotaAccounting {
accounting := benchmarkQuotaAccounting{billingSource: codex.BenchmarkBillingUnknown}
if provider, ok := runner.(benchmarkBillingProvider); ok {
accounting.billingSource = provider.BenchmarkBillingSource()
}
return accounting
}

func (a *benchmarkQuotaAccounting) start() {
a.runSequence++
a.active = a.billingSource == codex.BenchmarkBillingSubscription
a.settling = false
a.accounted = make(map[string]struct{})
if a.active {
a.revision++
}
}

func (a *benchmarkQuotaAccounting) finish() bool {
wasActive := a.active
a.active = false
if wasActive {
a.settling = true
a.revision++
}
return a.settling
}

func (a *benchmarkQuotaAccounting) settle() {
a.settling = false
}

func (a *benchmarkQuotaAccounting) abandonActiveResult() {
if a.billingSource != codex.BenchmarkBillingSubscription || !a.active {
return
}
a.unpricedCalls++
a.revision++
}

func (a *benchmarkQuotaAccounting) observe(result codex.BenchmarkResult) {
if a.billingSource != codex.BenchmarkBillingSubscription {
return
}
key := fmt.Sprintf("%d\x00%s", a.runSequence, benchmarkRunKey(result))
if _, exists := a.accounted[key]; exists {
return
}
a.accounted[key] = struct{}{}

cost, known := benchmarkQuotaCost(result)
calls := int64(len(result.ResponseUsage))
if calls == 0 {
calls = 1
}
if known {
a.costUSD += cost
a.pricedCalls += calls
a.revision++
return
}
a.unpricedCalls += calls
a.revision++
}

func benchmarkQuotaCost(result codex.BenchmarkResult) (float64, bool) {
if result.CostKnown && validBenchmarkQuotaCost(result.CostUSD) {
return result.CostUSD, true
}
model := strings.TrimSpace(result.ActualModel)
if model == "" {
model = result.Model
}
if !result.UsageKnown {
return 0, false
}
if result.UsageSource == codex.BenchmarkUsageRawResponses && len(result.ResponseUsage) > 0 {
var total float64
for _, response := range result.ResponseUsage {
cost, known, _ := codex.EstimateStandardAPIEqCost(model, response.Usage)
if !known {
return 0, false
}
total += cost
}
return total, validBenchmarkQuotaCost(total)
}
cost, known, _ := codex.EstimateStandardAPIEqAggregateCost(model, result.Usage)
return cost, known && validBenchmarkQuotaCost(cost)
}

func validBenchmarkQuotaCost(cost float64) bool {
return cost >= 0 && !math.IsNaN(cost) && !math.IsInf(cost, 0)
}

func (a benchmarkQuotaAccounting) combine(usage codex.LiveUsageSnapshot) codex.LiveUsageSnapshot {
if a.billingSource != codex.BenchmarkBillingSubscription {
return usage
}
usage.APIEqUSD += a.costUSD
usage.APIEqPricedCalls += a.pricedCalls
usage.APIEqUnpricedCalls += a.unpricedCalls
if a.active {
usage.APIEqPendingCalls++
}
return usage
}

func (a benchmarkQuotaAccounting) deferred() bool {
return a.billingSource == codex.BenchmarkBillingSubscription && (a.active || a.settling)
}

func (a benchmarkQuotaAccounting) deferredLabel(width int) string {
if a.active {
full, compact := "API-EQ DEFERRED // SUBSCRIPTION BENCHMARK ACTIVE", "API-EQ // BENCHMARK ACTIVE"
if width >= len(full) {
return full
}
if width >= len(compact) {
return compact
}
return "EQ // BENCH ACTIVE"
}
full, compact := "API-EQ DEFERRED // BENCHMARK ACCOUNTING SETTLING", "API-EQ // BENCHMARK SETTLING"
if width >= len(full) {
return full
}
if width >= len(compact) {
return compact
}
return "EQ // BENCH SETTLE"
}
Loading