feat(bench): add local-coder benchmark harness

Makefile

@@ -139,7 +139,7 @@ package: build ## Build a Linux/macOS tarball package
 		"$(DIST_DIR)/$(PACKAGE_NAME)/packaging/systemd" \
 		"$(DIST_DIR)/$(PACKAGE_NAME)/packaging/linux"
 	cp configs/router.example.yaml "$(DIST_DIR)/$(PACKAGE_NAME)/configs/"
-	cp docs/architecture.md docs/packaging.md docs/roadmap.md "$(DIST_DIR)/$(PACKAGE_NAME)/docs/"
+	cp docs/architecture.md docs/benchmarking.md docs/packaging.md docs/roadmap.md "$(DIST_DIR)/$(PACKAGE_NAME)/docs/"
 	cp packaging/systemd/devrail-router.service "$(DIST_DIR)/$(PACKAGE_NAME)/packaging/systemd/"
 	cp packaging/linux/install.sh "$(DIST_DIR)/$(PACKAGE_NAME)/packaging/linux/"
 	chmod 0755 "$(DIST_DIR)/$(PACKAGE_NAME)/devrail-router" \

README.md

@@ -34,6 +34,7 @@ This repository is in early foundation work. The current service supports:
   first event latency, bytes, and token totals
 - consistent OpenAI-shaped errors for router-side failures
 - request IDs in router responses and logs
+- a streamed benchmark harness for comparing model aliases with fixed prompts
 
 Routing policy, auth, native LM Studio lifecycle integration, richer telemetry,
 and Omarchy integration are planned next. See [docs/roadmap.md](docs/roadmap.md).
@@ -100,6 +101,17 @@ The Compose stack starts DevRail Router plus a mock OpenAI-compatible backend
 and verifies health, model listing, alias rewriting, backend auth injection, and
 chat completion proxying.
 
+Run repeatable streamed benchmark cases against a router alias:
+
+```sh
+go run ./cmd/devrail-router bench \
+  -base-url http://llm-srv-01.mfsoho.linkridge.net:18080/v1 \
+  -model local-coder \
+  -cases test/bench/local-coder.cases.json
+```
+
+See [docs/benchmarking.md](docs/benchmarking.md).
+
 ## Configuration
 
 See `configs/router.example.yaml`.

cmd/devrail-router/main.go

@@ -11,6 +11,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/devrail-dev/devrail-router/internal/bench"
 	"github.com/devrail-dev/devrail-router/internal/config"
 	"github.com/devrail-dev/devrail-router/internal/server"
 )
@@ -31,6 +32,8 @@ func run(args []string) int {
 		return serve(args[1:])
 	case "check":
 		return check(args[1:])
+	case "bench":
+		return runBench(args[1:])
 	case "version":
 		fmt.Println(version)
 		return 0
@@ -97,6 +100,32 @@ func serve(args []string) int {
 	return 0
 }
 
+func runBench(args []string) int {
+	fs := flag.NewFlagSet("bench", flag.ContinueOnError)
+	baseURL := fs.String("base-url", "http://127.0.0.1:8080/v1", "OpenAI-compatible base URL")
+	model := fs.String("model", "local-coder", "model alias to benchmark")
+	casesPath := fs.String("cases", "test/bench/local-coder.cases.json", "JSON benchmark cases file")
+	apiKey := fs.String("api-key", os.Getenv("OPENAI_API_KEY"), "API key for the target endpoint")
+	maxTokens := fs.Int("max-tokens", 512, "max completion tokens per case")
+	timeout := fs.Duration("timeout", 5*time.Minute, "timeout per benchmark case")
+	if err := fs.Parse(args); err != nil {
+		return 2
+	}
+
+	if err := bench.Run(context.Background(), bench.Options{
+		BaseURL:   *baseURL,
+		Model:     *model,
+		APIKey:    *apiKey,
+		CasesPath: *casesPath,
+		MaxTokens: *maxTokens,
+		Timeout:   *timeout,
+	}); err != nil {
+		slog.Error("benchmark failed", "error", err)
+		return 1
+	}
+	return 0
+}
+
 func check(args []string) int {
 	fs := flag.NewFlagSet("check", flag.ContinueOnError)
 	configPath := fs.String("config", config.DefaultPath, "path to router config")
@@ -120,6 +149,7 @@ func usage() {
 Usage:
   devrail-router serve [-config path]
   devrail-router check [-config path]
+  devrail-router bench [-base-url url] [-model alias] [-cases path]
   devrail-router version
 
 `, version)

docs/benchmarking.md

@@ -0,0 +1,76 @@
+# Benchmarking
+
+DevRail Router includes a small streamed benchmark harness for comparing
+OpenAI-compatible model aliases such as `local-coder`.
+
+The harness sends fixed chat-completion cases with `stream=true` and
+`stream_options.include_usage=true`, then writes one JSON object per case to
+stdout. Each result captures:
+
+- case ID
+- model alias
+- router request ID, when returned
+- HTTP status
+- time to first SSE event
+- total request duration
+- response bytes
+- prompt, completion, and total tokens when the backend emits streamed usage
+- a short first-content sample for sanity checking
+
+## Local-Coder Baseline
+
+Run the default coding-oriented cases against the llm-srv router:
+
+```sh
+go run ./cmd/devrail-router bench \
+  -base-url http://llm-srv-01.mfsoho.linkridge.net:18080/v1 \
+  -model local-coder \
+  -cases test/bench/local-coder.cases.json \
+  -max-tokens 512
+```
+
+Save a baseline:
+
+```sh
+go run ./cmd/devrail-router bench \
+  -base-url http://llm-srv-01.mfsoho.linkridge.net:18080/v1 \
+  -model local-coder \
+  -cases test/bench/local-coder.cases.json \
+  -max-tokens 512 \
+  > local-coder-baseline.jsonl
+```
+
+Run the same cases against another alias, such as an experimental
+`local-coder-parallel`, by changing `-model` only. Keeping the case file and
+token cap stable makes queue wait, first-token latency, duration, and token
+throughput easier to compare in Grafana.
+
+## Custom Cases
+
+Case files are JSON arrays. A case can use a simple `prompt`:
+
+```json
+[
+  {
+    "id": "small-refactor",
+    "prompt": "Refactor this Go function and explain the tradeoff."
+  }
+]
+```
+
+Or an explicit OpenAI-style message list:
+
+```json
+[
+  {
+    "id": "reviewer",
+    "messages": [
+      {"role": "system", "content": "You are a concise Go reviewer."},
+      {"role": "user", "content": "Find the highest-risk bug in this proxy."}
+    ]
+  }
+]
+```
+
+Use stable, short IDs. They appear in JSONL output and make it easier to line
+up command results with router request IDs, logs, and Prometheus samples.