diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ef786d..4d4c438 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Opt-in command-backed model profile ensure hooks. +- Roadmap for maturing DevRail Router from a single-backend gateway into an + observable local inference control plane. +- Request IDs are now added to router responses and structured logs. + +### Changed + +- Router-originated proxy and request validation failures now return + OpenAI-shaped JSON errors consistently. ## [1.0.0] - 2026-03-01 diff --git a/Makefile b/Makefile index e76c593..2385688 100644 --- a/Makefile +++ b/Makefile @@ -139,7 +139,7 @@ package: build ## Build a Linux/macOS tarball package "$(DIST_DIR)/$(PACKAGE_NAME)/packaging/systemd" \ "$(DIST_DIR)/$(PACKAGE_NAME)/packaging/linux" cp configs/router.example.yaml "$(DIST_DIR)/$(PACKAGE_NAME)/configs/" - cp docs/architecture.md docs/packaging.md "$(DIST_DIR)/$(PACKAGE_NAME)/docs/" + cp docs/architecture.md docs/packaging.md docs/roadmap.md "$(DIST_DIR)/$(PACKAGE_NAME)/docs/" cp packaging/systemd/devrail-router.service "$(DIST_DIR)/$(PACKAGE_NAME)/packaging/systemd/" cp packaging/linux/install.sh "$(DIST_DIR)/$(PACKAGE_NAME)/packaging/linux/" chmod 0755 "$(DIST_DIR)/$(PACKAGE_NAME)/devrail-router" \ diff --git a/README.md b/README.md index 2e6ab93..46edebb 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,12 @@ This repository is in early foundation work. The current service supports: - Linux tarball packaging - Linux/systemd install script and unit - Docker image and Compose smoke testing with a mock OpenAI-compatible backend +- response telemetry for proxied backend calls +- consistent OpenAI-shaped errors for router-side failures +- request IDs in router responses and logs -Routing policy, auth, telemetry, native LM Studio lifecycle integration, and -Omarchy integration are planned next. +Routing policy, auth, native LM Studio lifecycle integration, richer telemetry, +and Omarchy integration are planned next. See [docs/roadmap.md](docs/roadmap.md). Model aliases can also set basic concurrency guardrails with `max_concurrent_requests`, `max_queue_size`, and `queue_timeout`. This lets heavy diff --git a/docs/roadmap.md b/docs/roadmap.md new file mode 100644 index 0000000..b7995bb --- /dev/null +++ b/docs/roadmap.md @@ -0,0 +1,112 @@ +# Roadmap + +DevRail Router is the local-first control plane between agent clients and +private inference backends. The near-term goal is not to become a full model +server. It is to make local inference predictable enough that coding agents and +ops assistants can use it without every client learning backend-specific +lifecycle, queueing, telemetry, and safety behavior. + +## Principles + +- Keep the public client surface OpenAI-compatible. +- Prefer explicit aliases over hidden routing behavior. +- Make backend state visible before adding smarter routing. +- Fail with useful OpenAI-shaped errors instead of hanging clients. +- Keep host-specific lifecycle code replaceable by native adapters later. +- Treat local GPUs as shared infrastructure, not disposable accelerators. + +## Phase 1: Reliable Single-Backend Gateway + +Status: in progress. + +This phase makes one local backend dependable for day-to-day tools such as +opencode, OpenClaw, and Hermes. + +- Stable `/healthz` and `/v1/models` endpoints. +- Alias-to-target model rewriting. +- Bounded per-alias queueing and concurrency limits. +- Command-backed profile ensure hooks for LM Studio and similar hosts. +- Response telemetry for route, status, duration, bytes, and usage tokens. +- Linux tarball packaging with systemd installation. +- Docker and Vagrant smoke tests. +- Consistent OpenAI-shaped router errors. +- Request IDs in responses and router logs. + +Useful next work: + +- Add readiness checks for configured backends. +- Add configurable upstream transport timeouts. +- Capture streaming completion telemetry without buffering streams. +- Publish example configs for common LM Studio, Ollama, and vLLM setups. + +## Phase 2: Native Backend Adapters + +Status: planned. + +Command hooks are a good bridge, but the router needs native adapters for +backends that expose enough local state. + +- LM Studio adapter: inspect loaded model, context length, parallel slots, TTL, + and active slot state. +- Ollama adapter: inspect model availability, keepalive policy, and active load. +- vLLM/SGLang adapter: expose server readiness, model limits, batching state, + and GPU pressure where available. +- Adapter-level decisions for passive JIT load, explicit profile load, or + `503` when a backend is busy switching. +- Backend lifecycle telemetry: load duration, unload events, profile mismatch, + and rejected switches. + +## Phase 3: Operator Telemetry And Evals + +Status: planned. + +The router should make quality and performance decisions measurable instead of +vibe-based. + +- Structured request logs with request ID, alias, target model, upstream model, + queue wait, ensure duration, first-token latency, total duration, status, and + token usage. +- Optional local JSONL telemetry sink for offline analysis. +- Small repeatable eval harness for real agent workflows: + - commit hook failure handling + - push and remote-branch verification + - MR/PR creation and status checks + - CI failure parsing + - long-context repo editing + - security and SRE review prompts +- Model scorecards that combine quality, latency, throughput, memory use, and + failure modes. + +## Phase 4: Policy Routing + +Status: planned. + +Once telemetry and evals exist, aliases can become policy-backed instead of +hard-coded to one model. + +- `local-coder-auto` route that selects a small, large, or cloud-approved model + based on prompt size, requested tools, risk class, and current backend state. +- Deterministic routing policies that are explainable in logs. +- Budget and privacy gates for optional remote fallbacks. +- Second-pass review workflows where a stronger model audits selected outputs. +- Graceful degradation when local GPUs are hot, busy, or offline. + +## Phase 5: Production Operations + +Status: planned. + +This phase turns the router from a lab service into durable local infrastructure. + +- Auth for non-loopback deployments. +- Rate limits by client, alias, or token. +- Admin endpoint or CLI for draining, reloading config, and inspecting state. +- Prometheus metrics. +- Signed release artifacts and package repository support. +- Upgrade and rollback runbooks. +- Omarchy and desktop integration profiles. + +## Current Bias + +Prioritize observability, explicit profiles, and evals before automatic routing. +The project should earn trust by showing exactly what happened for each request +before it starts making hidden model-selection decisions. diff --git a/internal/server/server.go b/internal/server/server.go index 2a93474..4ab2601 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -3,6 +3,8 @@ package server import ( "bytes" "context" + "crypto/rand" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -98,25 +100,28 @@ func (s *Server) handleModels(w http.ResponseWriter, _ *http.Request) { } func (s *Server) proxyOpenAI(w http.ResponseWriter, r *http.Request) { + requestID := devrailRequestID(r) + w.Header().Set("X-Devrail-Request-ID", requestID) + modelID, body, err := requestModel(r) if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + writeOpenAIError(w, http.StatusBadRequest, err.Error(), "invalid_request_error", "invalid_request") return } model, ok := s.cfg.Model(modelID) if !ok { - http.Error(w, fmt.Sprintf("unknown model alias %q", modelID), http.StatusBadRequest) + writeOpenAIError(w, http.StatusBadRequest, fmt.Sprintf("unknown model alias %q", modelID), "invalid_request_error", "unknown_model_alias") return } backend, ok := s.cfg.Backend(model.Backend) if !ok { - http.Error(w, fmt.Sprintf("unknown backend %q", model.Backend), http.StatusInternalServerError) + writeOpenAIError(w, http.StatusInternalServerError, fmt.Sprintf("unknown backend %q", model.Backend), "devrail_config_error", "unknown_backend") return } - release, ok := s.acquireModelSlot(w, r, model) + release, ok := s.acquireModelSlot(w, r, model, requestID) if !ok { return } @@ -124,13 +129,13 @@ func (s *Server) proxyOpenAI(w http.ResponseWriter, r *http.Request) { defer release() } - if !s.ensureModelReady(w, r, model) { + if !s.ensureModelReady(w, r, model, requestID) { return } body, err = rewriteModel(body, model.TargetModel) if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + writeOpenAIError(w, http.StatusBadRequest, err.Error(), "invalid_request_error", "invalid_request") return } @@ -140,7 +145,7 @@ func (s *Server) proxyOpenAI(w http.ResponseWriter, r *http.Request) { target, err := url.Parse(backend.BaseURL) if err != nil { - http.Error(w, "backend base_url is invalid", http.StatusInternalServerError) + writeOpenAIError(w, http.StatusInternalServerError, "backend base_url is invalid", "devrail_config_error", "invalid_backend_base_url") return } @@ -154,7 +159,7 @@ func (s *Server) proxyOpenAI(w http.ResponseWriter, r *http.Request) { setBackendAuth(req, backend) } proxy.ModifyResponse = func(resp *http.Response) error { - instrumentBackendResponse(resp, started, model, backend) + instrumentBackendResponse(resp, started, model, backend, requestID) return nil } proxy.ErrorHandler = func(rw http.ResponseWriter, req *http.Request, proxyErr error) { @@ -162,20 +167,21 @@ func (s *Server) proxyOpenAI(w http.ResponseWriter, r *http.Request) { "backend request failed", "method", req.Method, "path", req.URL.Path, + "request_id", requestID, "alias", model.ID, "target_model", model.TargetModel, "backend", backend.ID, "duration_ms", time.Since(started).Milliseconds(), "error", proxyErr, ) - http.Error(rw, "backend request failed", http.StatusBadGateway) + writeOpenAIError(rw, http.StatusBadGateway, "backend request failed", "devrail_backend_error", "backend_request_failed") } - slog.Info("routing request", "alias", model.ID, "target_model", model.TargetModel, "backend", backend.ID) + slog.Info("routing request", "request_id", requestID, "alias", model.ID, "target_model", model.TargetModel, "backend", backend.ID) proxy.ServeHTTP(w, r) } -func (s *Server) acquireModelSlot(w http.ResponseWriter, r *http.Request, model config.ModelConfig) (func(), bool) { +func (s *Server) acquireModelSlot(w http.ResponseWriter, r *http.Request, model config.ModelConfig, requestID string) (func(), bool) { limiter, ok := s.limiters[model.ID] if !ok { return nil, true @@ -186,6 +192,7 @@ func (s *Server) acquireModelSlot(w http.ResponseWriter, r *http.Request, model w.Header().Set("X-Devrail-Queue-Wait-Ms", fmt.Sprintf("%d", waited.Milliseconds())) slog.Info( "acquired model slot", + "request_id", requestID, "alias", model.ID, "active", snapshot.active, "queued", snapshot.queued, @@ -205,6 +212,7 @@ func (s *Server) acquireModelSlot(w http.ResponseWriter, r *http.Request, model slog.Warn( "rejected queued request", + "request_id", requestID, "alias", model.ID, "active", snapshot.active, "queued", snapshot.queued, @@ -214,7 +222,7 @@ func (s *Server) acquireModelSlot(w http.ResponseWriter, r *http.Request, model return nil, false } -func (s *Server) ensureModelReady(w http.ResponseWriter, r *http.Request, model config.ModelConfig) bool { +func (s *Server) ensureModelReady(w http.ResponseWriter, r *http.Request, model config.ModelConfig, requestID string) bool { if model.Ensure.Mode == "" || model.Ensure.Mode == "disabled" { return true } @@ -229,14 +237,14 @@ func (s *Server) ensureModelReady(w http.ResponseWriter, r *http.Request, model switch model.Ensure.Mode { case "command": - return s.ensureModelReadyWithCommand(w, r, model) + return s.ensureModelReadyWithCommand(w, r, model, requestID) default: writeOpenAIError(w, http.StatusInternalServerError, "model ensure mode is unsupported", "devrail_ensure_unsupported", "ensure_unsupported") return false } } -func (s *Server) ensureModelReadyWithCommand(w http.ResponseWriter, r *http.Request, model config.ModelConfig) bool { +func (s *Server) ensureModelReadyWithCommand(w http.ResponseWriter, r *http.Request, model config.ModelConfig, requestID string) bool { timeout, err := model.Ensure.TimeoutDuration() if err != nil { writeOpenAIError(w, http.StatusInternalServerError, "model ensure timeout is invalid", "devrail_ensure_config_error", "ensure_config_error") @@ -264,6 +272,7 @@ func (s *Server) ensureModelReadyWithCommand(w http.ResponseWriter, r *http.Requ slog.Warn( "model ensure command failed", + "request_id", requestID, "alias", model.ID, "target_model", model.TargetModel, "error", err, @@ -275,6 +284,7 @@ func (s *Server) ensureModelReadyWithCommand(w http.ResponseWriter, r *http.Requ slog.Info( "model ensure command completed", + "request_id", requestID, "alias", model.ID, "target_model", model.TargetModel, "output", outputText, @@ -283,6 +293,7 @@ func (s *Server) ensureModelReadyWithCommand(w http.ResponseWriter, r *http.Requ } type responseTelemetry struct { + RequestID string Alias string TargetModel string Backend string @@ -320,6 +331,7 @@ func (body *telemetryReadCloser) log() { body.once.Do(func() { slog.Info( "backend response completed", + "request_id", body.telemetry.RequestID, "alias", body.telemetry.Alias, "target_model", body.telemetry.TargetModel, "upstream_model", body.telemetry.UpstreamModel, @@ -334,12 +346,13 @@ func (body *telemetryReadCloser) log() { }) } -func instrumentBackendResponse(resp *http.Response, started time.Time, model config.ModelConfig, backend config.BackendConfig) { +func instrumentBackendResponse(resp *http.Response, started time.Time, model config.ModelConfig, backend config.BackendConfig, requestID string) { if resp.Body == nil { return } telemetry := &responseTelemetry{ + RequestID: requestID, Alias: model.ID, TargetModel: model.TargetModel, Backend: backend.ID, @@ -393,6 +406,22 @@ func applyOpenAIUsageTelemetry(raw []byte, telemetry *responseTelemetry) { telemetry.TotalTokens = payload.Usage.TotalTokens } +func devrailRequestID(r *http.Request) string { + for _, header := range []string{"X-Devrail-Request-ID", "X-Request-ID"} { + value := strings.TrimSpace(r.Header.Get(header)) + if value != "" { + return value + } + } + + var raw [16]byte + if _, err := rand.Read(raw[:]); err == nil { + return hex.EncodeToString(raw[:]) + } + + return fmt.Sprintf("%d", time.Now().UnixNano()) +} + func requestModel(r *http.Request) (string, []byte, error) { if r.Body == nil { return "", nil, fmt.Errorf("request body is required") diff --git a/internal/server/server_test.go b/internal/server/server_test.go index b8b70b5..28c36a2 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -63,6 +63,65 @@ func TestUnknownAliasReturnsBadRequest(t *testing.T) { if rec.Code != http.StatusBadRequest { t.Fatalf("unexpected status: %d", rec.Code) } + assertOpenAIErrorCode(t, rec.Body.Bytes(), "unknown_model_alias") +} + +func TestMalformedRequestReturnsOpenAIError(t *testing.T) { + t.Parallel() + + srv := testServer(t) + req := httptest.NewRequest( + http.MethodPost, + "/v1/chat/completions", + strings.NewReader(`{"messages":[]}`), + ) + rec := httptest.NewRecorder() + + srv.ServeHTTP(rec, req) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("unexpected status: %d", rec.Code) + } + if rec.Header().Get("X-Devrail-Request-ID") == "" { + t.Fatal("expected request id response header") + } + assertOpenAIErrorCode(t, rec.Body.Bytes(), "invalid_request") +} + +func TestRequestIDHeaderIsPropagated(t *testing.T) { + t.Parallel() + + var backendRequestID string + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + backendRequestID = r.Header.Get("X-Request-ID") + writeJSON(w, http.StatusOK, map[string]string{"model": "target-model"}) + })) + t.Cleanup(backend.Close) + + srv := testServerWithBackend(t, backend.URL, config.ModelConfig{ + ID: "local-coder", + Backend: "lmstudio", + TargetModel: "target-model", + }) + req := httptest.NewRequest( + http.MethodPost, + "/v1/chat/completions", + strings.NewReader(`{"model":"local-coder","messages":[]}`), + ) + req.Header.Set("X-Request-ID", "test-request-id") + rec := httptest.NewRecorder() + + srv.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("unexpected status: %d", rec.Code) + } + if got := rec.Header().Get("X-Devrail-Request-ID"); got != "test-request-id" { + t.Fatalf("unexpected response request id: %q", got) + } + if backendRequestID != "test-request-id" { + t.Fatalf("unexpected backend request id: %q", backendRequestID) + } } func TestJoinOpenAIPathAvoidsDuplicateVersionPrefix(t *testing.T) { @@ -381,6 +440,7 @@ func TestBackendResponseTelemetryLogsUsage(t *testing.T) { logText := logs.String() for _, want := range []string{ `"msg":"backend response completed"`, + `"request_id":`, `"alias":"local-coder"`, `"target_model":"target-model"`, `"upstream_model":"target-model"`, @@ -395,6 +455,29 @@ func TestBackendResponseTelemetryLogsUsage(t *testing.T) { } } +func TestBackendProxyErrorReturnsOpenAIError(t *testing.T) { + t.Parallel() + + srv := testServerWithBackend(t, "http://127.0.0.1:1/v1", config.ModelConfig{ + ID: "local-coder", + Backend: "lmstudio", + TargetModel: "target-model", + }) + req := httptest.NewRequest( + http.MethodPost, + "/v1/chat/completions", + strings.NewReader(`{"model":"local-coder","messages":[]}`), + ) + rec := httptest.NewRecorder() + + srv.ServeHTTP(rec, req) + + if rec.Code != http.StatusBadGateway { + t.Fatalf("unexpected status: %d", rec.Code) + } + assertOpenAIErrorCode(t, rec.Body.Bytes(), "backend_request_failed") +} + func testServer(t *testing.T) *Server { t.Helper() @@ -437,3 +520,19 @@ func serveChat(t *testing.T, srv *Server, model string) int { _, _ = io.Copy(io.Discard, rec.Result().Body) return rec.Code } + +func assertOpenAIErrorCode(t *testing.T, raw []byte, want string) { + t.Helper() + + var payload struct { + Error struct { + Code string `json:"code"` + } `json:"error"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + t.Fatalf("decode error response: %v\nbody: %s", err, string(raw)) + } + if payload.Error.Code != want { + t.Fatalf("unexpected error code: %q, want %q", payload.Error.Code, want) + } +}