Skip to content

fix: launcher restart policy + complete nginx lazy-resolution migration #77

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
man4ish merged 1 commit into from
Sep 2, 2026
Merged

fix: launcher restart policy + complete nginx lazy-resolution migration #77

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,8 @@ migration_verification_report.txt
STALE_SERVICES.txt
.aider*
STALE_SERVICES.txt

# Dated pre-change backups of live config (e.g. docker/nginx-router.conf.bak-*)
# -- git history on the real file is the rollback mechanism now, these are
# just local working-copy scratch files.
docker/*.bak-*
1 change: 1 addition & 0 deletions docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1128,6 +1128,7 @@ services:
# to it.
- docker-proxy-socket:/var/run/proxy-socket:ro
- /usr/bin/docker:/usr/bin/docker:ro
restart: unless-stopped
depends_on:
docker-socket-proxy:
condition: service_started
Expand Down
131 changes: 64 additions & 67 deletions docker/nginx-router.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,9 @@
# (this file is `include`d inside nginx.conf's http{} block) so every
# server{} in this file shares one resolver instead of repeating it.
# A static `upstream auth { server auth-service:8001; }` block (the old
# form, still used below for services this pass didn't touch) resolves its
# hostname exactly once, at nginx startup/reload -- if that backend
# form -- no longer used anywhere in this file, see the 2026-09-02 note
# below) resolves its hostname exactly once, at nginx startup/reload --
# if that backend
# container is later recreated and gets a new Docker-network IP (a plain
# restart, a redeploy, anything short of restarting nginx-router itself),
# nginx keeps sending traffic to the old, now-dead IP and every request
Expand Down Expand Up @@ -140,35 +141,17 @@ map $http_authorization $control_authorization {
'' $control_cookie_authorization;
}

# workbench, lims, auth, policy-engine, and hpc-policy-engine deliberately
# have NO static upstream {} block here anymore -- see the resolver comment
# above. Every location that used to `proxy_pass http://<name>;` now does
# 2026-09-02: migration finished -- every location in this file now uses
# `set $<name>_upstream <service>:<port>; proxy_pass http://$<name>_upstream;`
# instead, same pattern as $billing_upstream below.
upstream rag { server rag:8096; }
upstream gateway { server api-gateway:8080; }
upstream audit { server security-audit:8004; }
upstream control { server control-center:7070; }
upstream grafana { server grafana:3000; }
upstream videos { server videos:8086; }
upstream launcher { server launcher:5190; }
upstream modelregistry { server model-registry:8095; }
upstream modelregistry-ui { server model-registry:5176; }
upstream workflows { server workflow-bundles:8098; }
upstream workflows-ui { server workflow-bundles:5178; }
upstream toolimages { server tool-images:8097; }
upstream toolimages-ui { server tool-images:5179; }
upstream devhub { server dev-hub:8082; }
upstream devhub-ui { server dev-hub:5173; }
upstream tes-ui { server tes:5177; }
upstream tes-api { server tes:8081; }
upstream opa { server opa:8181; }
upstream toolserver { server toolserver:9090; }
upstream prometheus { server prometheus:9090; }
upstream jupyter { server jupyter:8888; }
upstream rstudio { server rstudio:8787; }
upstream vscode { server vscode:8080; }
upstream web-ui { server web-ui:80; }
# (see the resolver comment above for why). No static upstream {} block
# remains anywhere in this file; this incident is what finished it --
# a static `upstream launcher { server launcher:5190; }` block refused to
# resolve at nginx startup while the launcher container was down
# (unrelated crash, no restart policy -- see that incident's own report),
# which took down EVERY route in this file, not just launcher's own --
# nginx refuses to start at all if any static upstream{} hostname is
# unresolvable at load time. Backup of the pre-migration file kept at
# docker/nginx-router.conf.bak-2026-09-02-pre-lazy-resolution.

# Public documentation host. The same files remain available under /docs/ on
# the Studio host; this server makes docs.omnibioai.org a clean entry gate
Expand Down Expand Up @@ -254,8 +237,9 @@ server {
return 301 /_svc/prometheus/graph;
}
location ^~ /_svc/prometheus/ {
set $prometheus_upstream prometheus:9090;
rewrite ^/_svc/prometheus(/.*)$ $1 break;
proxy_pass http://prometheus;
proxy_pass http://$prometheus_upstream;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Accept-Encoding "";
Expand All @@ -265,7 +249,7 @@ server {
}

# ── Service routes ────────────────────────────────────────────────────────
location ^~ /_svc/gateway { rewrite ^/_svc/gateway(/.*)$ $1 break; proxy_pass http://gateway; }
location ^~ /_svc/gateway { set $gateway_upstream api-gateway:8080; rewrite ^/_svc/gateway(/.*)$ $1 break; proxy_pass http://$gateway_upstream; }
location ^~ /docs/ {
alias /docs/dist/;
index index.html;
Expand Down Expand Up @@ -341,7 +325,8 @@ server {
rewrite ^ /_svc/rag/ last;
}
location ^~ /_svc/rag/ {
proxy_pass http://rag/;
set $rag_upstream rag:8096;
proxy_pass http://$rag_upstream/;
proxy_set_header Authorization $http_authorization;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
Expand All @@ -356,7 +341,7 @@ server {
# `modelregistry`), 5176 is the actual UI, whose built assets are
# hardcoded to expect this exact /_svc/modelregistry/ prefix (see its
# index.html <script>/<link> src).
location ^~ /_svc/modelregistry { rewrite ^/_svc/modelregistry(/.*)$ $1 break; proxy_pass http://modelregistry-ui; }
location ^~ /_svc/modelregistry { set $modelregistry_ui_upstream model-registry:5176; rewrite ^/_svc/modelregistry(/.*)$ $1 break; proxy_pass http://$modelregistry_ui_upstream; }
# model-registry-ui's own source is internally inconsistent about its
# base path (two different bare-path families, confirmed live, both
# actually used):
Expand Down Expand Up @@ -388,20 +373,21 @@ server {
# not via nginx auth_request; resolve/verify/hf-settings need no auth
# at all. No WebSocket or streaming (SSE) endpoints here either.
location ^~ /modelregistry/v1 {
set $modelregistry_upstream model-registry:8095;
rewrite ^/modelregistry(/.*)$ $1 break;
proxy_pass http://modelregistry;
}
location ^~ /v1/hf { proxy_pass http://modelregistry; }
location = /v1/promote { proxy_pass http://modelregistry; }
location = /v1/register { proxy_pass http://modelregistry; }
location = /v1/resolve { proxy_pass http://modelregistry; }
location = /v1/verify { proxy_pass http://modelregistry; }
location ^~ /_svc/workflows/v1 { rewrite ^/_svc/workflows(/.*)$ $1 break; proxy_pass http://workflows; }
location ^~ /_svc/workflows/assets/ { proxy_pass http://workflows-ui; }
location ^~ /_svc/workflows { proxy_pass http://workflows-ui; }
proxy_pass http://$modelregistry_upstream;
}
location ^~ /v1/hf { set $modelregistry_upstream model-registry:8095; proxy_pass http://$modelregistry_upstream; }
location = /v1/promote { set $modelregistry_upstream model-registry:8095; proxy_pass http://$modelregistry_upstream; }
location = /v1/register { set $modelregistry_upstream model-registry:8095; proxy_pass http://$modelregistry_upstream; }
location = /v1/resolve { set $modelregistry_upstream model-registry:8095; proxy_pass http://$modelregistry_upstream; }
location = /v1/verify { set $modelregistry_upstream model-registry:8095; proxy_pass http://$modelregistry_upstream; }
location ^~ /_svc/workflows/v1 { set $workflows_upstream workflow-bundles:8098; rewrite ^/_svc/workflows(/.*)$ $1 break; proxy_pass http://$workflows_upstream; }
location ^~ /_svc/workflows/assets/ { set $workflows_ui_upstream workflow-bundles:5178; proxy_pass http://$workflows_ui_upstream; }
location ^~ /_svc/workflows { set $workflows_ui_upstream workflow-bundles:5178; proxy_pass http://$workflows_ui_upstream; }
# Same split as model-registry above: tool-images' UI is on 5179, its
# JSON API (upstream `toolimages`) is on 8097.
location ^~ /_svc/toolimages { rewrite ^/_svc/toolimages(/.*)$ $1 break; proxy_pass http://toolimages-ui; }
location ^~ /_svc/toolimages { set $toolimages_ui_upstream tool-images:5179; rewrite ^/_svc/toolimages(/.*)$ $1 break; proxy_pass http://$toolimages_ui_upstream; }
# tool-images-ui's own source (src/api.ts: `const BASE = ''`) calls
# these bare, root-relative — confirmed live, matching
# api/server.py's exact route set (no other /v1/* routes exist there).
Expand All @@ -411,13 +397,14 @@ server {
# streaming, same as dev-hub's /rag/stream), hence proxy_buffering off.
# Collision-audited against the other 4 frontends' source and every
# other backend's route definitions — zero hits.
location ^~ /v1/tools { proxy_pass http://toolimages; }
location ^~ /v1/tools { set $toolimages_upstream tool-images:8097; proxy_pass http://$toolimages_upstream; }
location ^~ /v1/build {
proxy_pass http://toolimages;
set $toolimages_upstream tool-images:8097;
proxy_pass http://$toolimages_upstream;
proxy_buffering off;
}
# dev-hub's UI is on 5173, its JSON API (upstream `devhub`) is on 8082.
location ^~ /_svc/devhub { rewrite ^/_svc/devhub(/.*)$ $1 break; proxy_pass http://devhub-ui; }
location ^~ /_svc/devhub { set $devhub_ui_upstream dev-hub:5173; rewrite ^/_svc/devhub(/.*)$ $1 break; proxy_pass http://$devhub_ui_upstream; }
# dev-hub-ui's own source (src/api/client.ts: `const API_BASE = ""`)
# calls these bare, root-relative, ignoring its own /_svc/devhub Vite
# base entirely — confirmed live in source, not just inferred from its
Expand Down Expand Up @@ -457,6 +444,7 @@ server {
# longer inherits the server-level ones — Host/X-Real-IP/X-Forwarded-For
# /X-Forwarded-Proto are re-declared explicitly below.
location ^~ /rag/ {
set $devhub_upstream dev-hub:8082;
auth_request /internal/auth/verify;
auth_request_set $auth_status $upstream_status;
error_page 401 = @devhub_unauthorized;
Expand All @@ -465,17 +453,17 @@ server {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $router_xfp;
proxy_set_header Authorization $control_authorization;
proxy_pass http://devhub;
proxy_pass http://$devhub_upstream;
proxy_buffering off;
}
location = /status { proxy_pass http://devhub; }
location ^~ /_svc/tes { rewrite ^/_svc/tes(/.*)$ $1 break; proxy_pass http://tes-ui; }
location = /status { set $devhub_upstream dev-hub:8082; proxy_pass http://$devhub_upstream; }
location ^~ /_svc/tes { set $tes_ui_upstream tes:5177; rewrite ^/_svc/tes(/.*)$ $1 break; proxy_pass http://$tes_ui_upstream; }
# TES's own dashboard UI is on 5177 (above); its JSON API (upstream
# `tes-api`) is on 8081 -- web-ui's Jobs.jsx fetches this bare via /_tes
# (issue #11), matching the vite dev-proxy table's /_tes entry.
location ^~ /_tes { rewrite ^/_tes(/.*)$ $1 break; proxy_pass http://tes-api; }
location ^~ /_svc/sdk { rewrite ^/_svc/sdk(/.*)$ $1 break; proxy_pass http://launcher; }
location ^~ /_svc/videos { rewrite ^/_svc/videos(/.*)$ $1 break; proxy_pass http://videos; }
location ^~ /_tes { set $tes_api_upstream tes:8081; rewrite ^/_tes(/.*)$ $1 break; proxy_pass http://$tes_api_upstream; }
location ^~ /_svc/sdk { set $launcher_upstream launcher:5190; rewrite ^/_svc/sdk(/.*)$ $1 break; proxy_pass http://$launcher_upstream; }
location ^~ /_svc/videos { set $videos_upstream videos:8086; rewrite ^/_svc/videos(/.*)$ $1 break; proxy_pass http://$videos_upstream; }
# /_svc/control with no trailing slash 404s: the rewrite below requires
# a "/" after "control" to fire, so the unmodified URI gets forwarded
# as-is to FastAPI, which has no route matching literal "/_svc/control"
Expand All @@ -489,17 +477,19 @@ server {
# Control center — public read-only endpoints (no JWT required)
# These are needed by the ecosystem report health tab and generate_report.py
location ~* ^/_svc/control/(health|summary|services|report|report/data)$ {
set $control_upstream control-center:7070;
rewrite ^/_svc/control(/.*)$ $1 break;
proxy_pass http://control;
proxy_pass http://$control_upstream;
}

# Control center — all other endpoints require JWT
location /_svc/control {
auth_request /internal/auth/verify;
auth_request_set $auth_status $upstream_status;
error_page 401 = @cc_unauthorized;
set $control_upstream control-center:7070;
rewrite ^/_svc/control(/.*)$ $1 break;
proxy_pass http://control;
proxy_pass http://$control_upstream;
}
# control-center's own served page (main.py's root(), NOT cc-ui — see
# the KNOWN DRIFT note at the top of this file for why cc-ui's own
Expand Down Expand Up @@ -532,7 +522,8 @@ server {
# externally-visible URL forever (nginx strips -> Grafana redirects to
# the prefixed URL -> nginx strips again -> loop).
location ^~ /_svc/monitor {
proxy_pass http://grafana;
set $grafana_upstream grafana:3000;
proxy_pass http://$grafana_upstream;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_pass_header Set-Cookie;
Expand All @@ -559,8 +550,9 @@ server {
auth_request_set $auth_status $upstream_status;
error_page 401 = @toolserver_unauthorized;
proxy_set_header Authorization $control_authorization;
set $toolserver_upstream toolserver:9090;
rewrite ^/_svc/toolserver(/.*)$ $1 break;
proxy_pass http://toolserver;
proxy_pass http://$toolserver_upstream;
}
location ^~ /_svc/auth {
limit_req zone=auth_limit burst=5 nodelay;
Expand Down Expand Up @@ -775,25 +767,27 @@ server {
proxy_set_header Connection "";
proxy_buffering off;
}
location ^~ /_svc/audit { rewrite ^/_svc/audit(/.*)$ $1 break; proxy_pass http://audit; }
location ^~ /_svc/opa { rewrite ^/_svc/opa(/.*)$ $1 break; proxy_pass http://opa; }
location ^~ /_svc/audit { set $audit_upstream security-audit:8004; rewrite ^/_svc/audit(/.*)$ $1 break; proxy_pass http://$audit_upstream; }
location ^~ /_svc/opa { set $opa_upstream opa:8181; rewrite ^/_svc/opa(/.*)$ $1 break; proxy_pass http://$opa_upstream; }

# ── IDE services ──────────────────────────────────────────────────────────
location /jupyter/ {
proxy_pass http://jupyter/;
set $jupyter_upstream jupyter:8888;
proxy_pass http://$jupyter_upstream/;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
location /rstudio/ { proxy_pass http://rstudio/; }
location /rstudio/ { set $rstudio_upstream rstudio:8787; proxy_pass http://$rstudio_upstream/; }
location /vscode/ {
proxy_pass http://vscode/;
set $vscode_upstream vscode:8080;
proxy_pass http://$vscode_upstream/;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}

location / { proxy_pass http://web-ui; }
location / { set $web_ui_upstream web-ui:80; proxy_pass http://$web_ui_upstream; }

# Internal subrequest — strips body, forwards Authorization header only
# (falling back to the access-token cookie via $control_authorization —
Expand Down Expand Up @@ -824,7 +818,8 @@ server {
# old accidental mechanism as if it were still the real one.
location = /internal/auth/verify {
internal;
proxy_pass http://gateway/auth/verify;
set $gateway_upstream api-gateway:8080;
proxy_pass http://$gateway_upstream/auth/verify;
proxy_pass_request_body off;
proxy_set_header Content-Length "";
proxy_set_header X-Original-URI $request_uri;
Expand Down Expand Up @@ -917,7 +912,8 @@ server {
# path reaches the same backend-generated main.py page and
# require_admin-gated routes that are already live today.
location / {
proxy_pass http://control;
set $control_upstream control-center:7070;
proxy_pass http://$control_upstream;
}
}

Expand Down Expand Up @@ -962,7 +958,8 @@ server {
# verification" reason as the /auth/ block above, ready for future use
# rather than another gap to close later.
location = /auth/verify {
proxy_pass http://gateway/auth/verify;
set $gateway_upstream api-gateway:8080;
proxy_pass http://$gateway_upstream/auth/verify;
}

# Everything else: LIMS's own container, unchanged. LIMS ships its own
Expand Down
Loading