From bc1c0565cd0c30f644439186ae62221095e96754 Mon Sep 17 00:00:00 2001 From: Jvst Me Date: Wed, 16 Sep 2026 23:11:31 +0200 Subject: [PATCH] Fix model endpoint protocol with LB-terminated TLS If the gateway has a load balancer with a TLS certificate (`acm`, `gcp-cm`), it always serves the model endpoint over HTTPS, yet the run's `service.model.base_url` used to incorrectly contain a `http://` URL. This commit fixes `service.model.base_url` so that it contains an `https://` URL in such cases. --- .../pipeline_tasks/gateway_replicas.py | 12 ++++++---- .../server/services/services/__init__.py | 24 +++++++++---------- .../server/services/services/test_services.py | 14 +++++------ 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py index ab8a7af11..44fd687cd 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py @@ -63,8 +63,8 @@ from dstack._internal.server.services.pipelines import PipelineHinterProtocol from dstack._internal.server.services.runs import get_run_spec from dstack._internal.server.services.services import ( - get_gateway_https, - should_configure_service_https_on_gateway, + should_configure_gateway_endpoint_https_on_gateway_replica, + should_configure_service_https_on_gateway_replica, ) from dstack._internal.server.utils import tracing from dstack._internal.utils.common import get_current_datetime, get_or_error, run_async @@ -1280,8 +1280,12 @@ async def _register_service( run_id=run_model.id, run_name=run_model.run_name, domain=domain, - service_https=should_configure_service_https_on_gateway(run_spec, gateway_configuration), - gateway_https=get_gateway_https(gateway_configuration), + service_https=should_configure_service_https_on_gateway_replica( + run_spec, gateway_configuration + ), + gateway_https=should_configure_gateway_endpoint_https_on_gateway_replica( + gateway_configuration + ), auth=run_spec.configuration.auth, client_max_body_size=settings.DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE, options=service_spec.options, diff --git a/src/dstack/_internal/server/services/services/__init__.py b/src/dstack/_internal/server/services/services/__init__.py index 91499880b..9b5868d8c 100644 --- a/src/dstack/_internal/server/services/services/__init__.py +++ b/src/dstack/_internal/server/services/services/__init__.py @@ -131,9 +131,6 @@ async def _assign_service_to_gateway( "Cannot run HTTPS service on gateway with no SSL certificates configured" ) - gateway_https = get_gateway_https(gateway_configuration) - gateway_protocol = "https" if gateway_https else "http" - wildcard_domain = gateway.wildcard_domain.lstrip("*.") if gateway.wildcard_domain else None if wildcard_domain is None: raise ServerClientError("Domain is required for gateway") @@ -146,6 +143,7 @@ async def _assign_service_to_gateway( if isinstance(run_spec.configuration.model, OpenAIChatModel): model_url = service_url + run_spec.configuration.model.prefix else: + gateway_protocol = "https" if gateway_configuration.certificate is not None else "http" model_url = f"{gateway_protocol}://gateway.{wildcard_domain}" service_spec = _get_service_spec( configuration=run_spec.configuration, @@ -220,11 +218,11 @@ def _get_service_spec( return service_spec -def should_configure_service_https_on_gateway( +def should_configure_service_https_on_gateway_replica( run_spec: RunSpec, configuration: GatewayConfiguration ) -> bool: """ - Returns `True` if the gateway needs to serve the service with HTTPS. + Returns `True` if the gateway replica needs to serve the service with HTTPS. May be `False` for HTTPS services, e.g. SSL termination is done on a load balancer. """ assert run_spec.configuration.type == "service" @@ -244,6 +242,14 @@ def should_configure_service_https_on_gateway( return True +def should_configure_gateway_endpoint_https_on_gateway_replica( + configuration: GatewayConfiguration, +) -> bool: + return ( + configuration.certificate is not None and configuration.certificate.type == "lets-encrypt" + ) + + def _should_show_service_https(run_spec: RunSpec, configuration: GatewayConfiguration) -> bool: """ Returns `True` if the service needs to be accessed via https://. @@ -257,11 +263,3 @@ def _should_show_service_https(run_spec: RunSpec, configuration: GatewayConfigur return False return True return https - - -def get_gateway_https(configuration: GatewayConfiguration) -> bool: - if is_tls_terminated_at_load_balancer(configuration.certificate): - return False - if configuration.certificate is not None and configuration.certificate.type == "lets-encrypt": - return True - return False diff --git a/src/tests/_internal/server/services/services/test_services.py b/src/tests/_internal/server/services/services/test_services.py index 27854893b..1ce03c52a 100644 --- a/src/tests/_internal/server/services/services/test_services.py +++ b/src/tests/_internal/server/services/services/test_services.py @@ -12,7 +12,7 @@ from dstack._internal.core.models.runs import RunSpec from dstack._internal.server.services.services import ( _should_show_service_https, - should_configure_service_https_on_gateway, + should_configure_service_https_on_gateway_replica, ) from dstack._internal.server.testing.common import get_run_spec @@ -55,36 +55,36 @@ class TestShouldConfigureServiceHttpsOnGateway: def test_auto_resolves_to_true_with_lets_encrypt_gateway(self) -> None: run_spec = _service_run_spec(https="auto") gw = _gateway_config(certificate=LetsEncryptGatewayCertificate()) - assert should_configure_service_https_on_gateway(run_spec, gw) is True + assert should_configure_service_https_on_gateway_replica(run_spec, gw) is True def test_auto_resolves_to_false_when_gateway_has_no_certificate(self) -> None: run_spec = _service_run_spec(https="auto") gw = _gateway_config(certificate=None) - assert should_configure_service_https_on_gateway(run_spec, gw) is False + assert should_configure_service_https_on_gateway_replica(run_spec, gw) is False def test_auto_resolves_to_false_with_acm_gateway(self) -> None: run_spec = _service_run_spec(https="auto") gw = _gateway_config( certificate=ACMGatewayCertificate(arn="arn:aws:acm:us-east-1:123:cert/abc") ) - assert should_configure_service_https_on_gateway(run_spec, gw) is False + assert should_configure_service_https_on_gateway_replica(run_spec, gw) is False def test_true_enables_https_when_gateway_has_no_certificate(self) -> None: run_spec = _service_run_spec(https=True) gw = _gateway_config(certificate=None) - assert should_configure_service_https_on_gateway(run_spec, gw) is True + assert should_configure_service_https_on_gateway_replica(run_spec, gw) is True def test_false_disables_https_regardless_of_gateway_certificate(self) -> None: run_spec = _service_run_spec(https=False) gw = _gateway_config(certificate=LetsEncryptGatewayCertificate()) - assert should_configure_service_https_on_gateway(run_spec, gw) is False + assert should_configure_service_https_on_gateway_replica(run_spec, gw) is False def test_true_does_not_configure_https_on_acm_gateway(self) -> None: run_spec = _service_run_spec(https=True) gw = _gateway_config( certificate=ACMGatewayCertificate(arn="arn:aws:acm:us-east-1:123:cert/abc") ) - assert should_configure_service_https_on_gateway(run_spec, gw) is False + assert should_configure_service_https_on_gateway_replica(run_spec, gw) is False class TestShouldShowServiceHttps: