diff --git a/.gitignore b/.gitignore index 1c9f70c..59a2da2 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,8 @@ migration_verification_report.txt STALE_SERVICES.txt .aider* STALE_SERVICES.txt + +# Dated pre-change backups of live config (e.g. docker/nginx-router.conf.bak-*) +# -- git history on the real file is the rollback mechanism now, these are +# just local working-copy scratch files. +docker/*.bak-* diff --git a/docker-compose.yml b/docker-compose.yml index 2fa29c2..05baec4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1128,6 +1128,7 @@ services: # to it. - docker-proxy-socket:/var/run/proxy-socket:ro - /usr/bin/docker:/usr/bin/docker:ro + restart: unless-stopped depends_on: docker-socket-proxy: condition: service_started diff --git a/docker/nginx-router.conf b/docker/nginx-router.conf index b7d582e..4955403 100644 --- a/docker/nginx-router.conf +++ b/docker/nginx-router.conf @@ -78,8 +78,9 @@ # (this file is `include`d inside nginx.conf's http{} block) so every # server{} in this file shares one resolver instead of repeating it. # A static `upstream auth { server auth-service:8001; }` block (the old -# form, still used below for services this pass didn't touch) resolves its -# hostname exactly once, at nginx startup/reload -- if that backend +# form -- no longer used anywhere in this file, see the 2026-09-02 note +# below) resolves its hostname exactly once, at nginx startup/reload -- +# if that backend # container is later recreated and gets a new Docker-network IP (a plain # restart, a redeploy, anything short of restarting nginx-router itself), # nginx keeps sending traffic to the old, now-dead IP and every request @@ -140,35 +141,17 @@ map $http_authorization $control_authorization { '' $control_cookie_authorization; } -# workbench, lims, auth, policy-engine, and hpc-policy-engine deliberately -# have NO static upstream {} block here anymore -- see the resolver comment -# above. Every location that used to `proxy_pass http://;` now does +# 2026-09-02: migration finished -- every location in this file now uses # `set $_upstream :; proxy_pass http://$_upstream;` -# instead, same pattern as $billing_upstream below. -upstream rag { server rag:8096; } -upstream gateway { server api-gateway:8080; } -upstream audit { server security-audit:8004; } -upstream control { server control-center:7070; } -upstream grafana { server grafana:3000; } -upstream videos { server videos:8086; } -upstream launcher { server launcher:5190; } -upstream modelregistry { server model-registry:8095; } -upstream modelregistry-ui { server model-registry:5176; } -upstream workflows { server workflow-bundles:8098; } -upstream workflows-ui { server workflow-bundles:5178; } -upstream toolimages { server tool-images:8097; } -upstream toolimages-ui { server tool-images:5179; } -upstream devhub { server dev-hub:8082; } -upstream devhub-ui { server dev-hub:5173; } -upstream tes-ui { server tes:5177; } -upstream tes-api { server tes:8081; } -upstream opa { server opa:8181; } -upstream toolserver { server toolserver:9090; } -upstream prometheus { server prometheus:9090; } -upstream jupyter { server jupyter:8888; } -upstream rstudio { server rstudio:8787; } -upstream vscode { server vscode:8080; } -upstream web-ui { server web-ui:80; } +# (see the resolver comment above for why). No static upstream {} block +# remains anywhere in this file; this incident is what finished it -- +# a static `upstream launcher { server launcher:5190; }` block refused to +# resolve at nginx startup while the launcher container was down +# (unrelated crash, no restart policy -- see that incident's own report), +# which took down EVERY route in this file, not just launcher's own -- +# nginx refuses to start at all if any static upstream{} hostname is +# unresolvable at load time. Backup of the pre-migration file kept at +# docker/nginx-router.conf.bak-2026-09-02-pre-lazy-resolution. # Public documentation host. The same files remain available under /docs/ on # the Studio host; this server makes docs.omnibioai.org a clean entry gate @@ -254,8 +237,9 @@ server { return 301 /_svc/prometheus/graph; } location ^~ /_svc/prometheus/ { + set $prometheus_upstream prometheus:9090; rewrite ^/_svc/prometheus(/.*)$ $1 break; - proxy_pass http://prometheus; + proxy_pass http://$prometheus_upstream; proxy_http_version 1.1; proxy_set_header Connection ""; proxy_set_header Accept-Encoding ""; @@ -265,7 +249,7 @@ server { } # ── Service routes ──────────────────────────────────────────────────────── - location ^~ /_svc/gateway { rewrite ^/_svc/gateway(/.*)$ $1 break; proxy_pass http://gateway; } + location ^~ /_svc/gateway { set $gateway_upstream api-gateway:8080; rewrite ^/_svc/gateway(/.*)$ $1 break; proxy_pass http://$gateway_upstream; } location ^~ /docs/ { alias /docs/dist/; index index.html; @@ -341,7 +325,8 @@ server { rewrite ^ /_svc/rag/ last; } location ^~ /_svc/rag/ { - proxy_pass http://rag/; + set $rag_upstream rag:8096; + proxy_pass http://$rag_upstream/; proxy_set_header Authorization $http_authorization; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; @@ -356,7 +341,7 @@ server { # `modelregistry`), 5176 is the actual UI, whose built assets are # hardcoded to expect this exact /_svc/modelregistry/ prefix (see its # index.html