From a56cad73af46f61842f3c4a0f2be152346486996 Mon Sep 17 00:00:00 2001 From: libops-agent <115990865+libops-agent@users.noreply.github.com> Date: Sat, 8 Aug 2026 12:01:12 +0000 Subject: [PATCH] [minor] Authenticate isolated Vault upstream --- README.md | 25 +++++- config.example.yaml | 5 ++ main.go | 187 ++++++++++++++++++++++++++++++++++++++++++-- main_test.go | 113 ++++++++++++++++++++++++++ 4 files changed, 322 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index a368525..83343eb 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,11 @@ # Vault proxy -Vault proxy is the public Cloud Run sidecar for the LibOps Vault service. Vault +Vault proxy is the public request boundary for the LibOps Vault service. Vault continues to authorize every Vault token and policy; the proxy adds a Google administrator check to routes that are not explicitly required by customer -authentication or secret access flows. +authentication or secret access flows. Production runs the proxy and Vault as +separate Cloud Run services and service identities so this public process does +not inherit Vault's storage or KMS authority. ## Request policy @@ -49,6 +51,25 @@ public_routes: - /v1/sys/health ``` +The localhost form is useful only for development or legacy sidecar use. For a +separate IAM-protected Cloud Run upstream, configure both fields with the +runtime service's default HTTPS URL: + +```yaml +vault_addr: https://vault-runtime-example.run.app +vault_audience: https://vault-runtime-example.run.app +``` + +When `vault_audience` is set, the upstream must use HTTPS. The proxy fetches a +short-lived Google-signed ID token from the Cloud Run metadata server, caches it +only until shortly before expiration, and sends it in +`X-Serverless-Authorization`. That header lets Cloud Run authenticate the proxy +identity while preserving a client's separate `Authorization` or +`X-Vault-Token` credential for Vault. Caller-supplied forwarding and +`X-Serverless-Authorization` headers are stripped before the proxy creates its +own upstream request. The proxy identity needs only `roles/run.invoker` on the +Vault runtime; it needs no Vault data-bucket or KMS role. + The example patterns permit OIDC discovery/exchange and user login while leaving role, provider, user, policy, and system management protected. Add secret-engine subtrees only when downstream Vault policies are intended to be diff --git a/config.example.yaml b/config.example.yaml index 8f92de7..59372c2 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -1,4 +1,9 @@ vault_addr: http://127.0.0.1:8200 +# For a separate, IAM-protected Cloud Run Vault service, use its HTTPS URL for +# both fields. The proxy fetches a short-lived ID token from the metadata server +# and sends it in X-Serverless-Authorization without replacing Vault's own +# Authorization header. +# vault_audience: https://vault-runtime-example.run.app port: 8080 admin_emails: - me@example.com diff --git a/main.go b/main.go index dcc3d84..00f4f6a 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "encoding/base64" "encoding/json" "errors" "flag" @@ -18,6 +19,7 @@ import ( "path" "strconv" "strings" + "sync" "syscall" "time" @@ -28,20 +30,26 @@ const ( defaultListenPort = 8080 // #nosec G101 -- this is Google's public token-inspection endpoint, not a credential. defaultTokenInfoURL = "https://oauth2.googleapis.com/tokeninfo" - maxTokenInfoBody = 1 << 20 + // #nosec G101 -- this is the documented metadata identity endpoint, not a credential. + defaultMetadataIdentityURL = "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity" + maxTokenInfoBody = 1 << 20 + maxIdentityTokenBody = 64 << 10 + identityTokenRefreshSkew = 5 * time.Minute ) // YAMLConfig represents the supported configuration file fields. type YAMLConfig struct { - VaultAddr string `yaml:"vault_addr"` - Port int `yaml:"port"` - AdminEmails []string `yaml:"admin_emails"` - PublicRoutes []string `yaml:"public_routes"` + VaultAddr string `yaml:"vault_addr"` + VaultAudience string `yaml:"vault_audience"` + Port int `yaml:"port"` + AdminEmails []string `yaml:"admin_emails"` + PublicRoutes []string `yaml:"public_routes"` } // Config is the validated runtime configuration. type Config struct { VaultTargetURL *url.URL + VaultAudience string PublicRoutes []string AdminEmails map[string]struct{} ListenPort int @@ -63,6 +71,24 @@ type googleTokenValidator struct { endpoint string } +type identityTokenProvider interface { + Token(context.Context, string) (string, error) +} + +type metadataIdentityTokenProvider struct { + client *http.Client + endpoint string + mu sync.Mutex + token string + expiresAt time.Time +} + +type identityTokenTransport struct { + base http.RoundTripper + provider identityTokenProvider + audience string +} + func loadYAMLConfig(configPath string) (*YAMLConfig, error) { var data []byte var err error @@ -106,6 +132,10 @@ func loadConfig(configPath string) (*Config, error) { if err != nil { return nil, err } + vaultAudience, err := validateVaultAudience(yamlConfig.VaultAudience, targetURL) + if err != nil { + return nil, err + } adminEmails := make(map[string]struct{}, len(yamlConfig.AdminEmails)) for _, configuredEmail := range yamlConfig.AdminEmails { @@ -136,12 +166,36 @@ func loadConfig(configPath string) (*Config, error) { return &Config{ VaultTargetURL: targetURL, + VaultAudience: vaultAudience, PublicRoutes: append([]string(nil), yamlConfig.PublicRoutes...), AdminEmails: adminEmails, ListenPort: port, }, nil } +func validateVaultAudience(rawAudience string, targetURL *url.URL) (string, error) { + if rawAudience == "" { + return "", nil + } + if strings.TrimSpace(rawAudience) != rawAudience { + return "", errors.New("vault_audience must not contain leading or trailing whitespace") + } + audience, err := url.Parse(rawAudience) + if err != nil { + return "", fmt.Errorf("invalid vault_audience: %w", err) + } + if audience.Scheme != "https" || audience.Host == "" { + return "", errors.New("vault_audience must be an absolute HTTPS service URL") + } + if audience.User != nil || audience.RawQuery != "" || audience.Fragment != "" || (audience.Path != "" && audience.Path != "/") { + return "", errors.New("vault_audience must not contain credentials, a path, query, or fragment") + } + if targetURL.Scheme != "https" { + return "", errors.New("vault_addr must use HTTPS when vault_audience enables upstream identity") + } + return strings.TrimSuffix(rawAudience, "/"), nil +} + func validateVaultURL(rawURL string) (*url.URL, error) { if strings.TrimSpace(rawURL) != rawURL || rawURL == "" { return nil, errors.New("vault_addr must be a non-empty absolute URL") @@ -265,6 +319,97 @@ func normalizeEmail(email string) string { return strings.ToLower(strings.TrimSpace(email)) } +func (provider *metadataIdentityTokenProvider) Token(ctx context.Context, audience string) (string, error) { + provider.mu.Lock() + defer provider.mu.Unlock() + + if provider.token != "" && time.Until(provider.expiresAt) > identityTokenRefreshSkew { + return provider.token, nil + } + endpoint, err := url.Parse(provider.endpoint) + if err != nil { + return "", errors.New("metadata identity endpoint is misconfigured") + } + query := endpoint.Query() + query.Set("audience", audience) + query.Set("format", "full") + endpoint.RawQuery = query.Encode() + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil) + if err != nil { + return "", errors.New("create metadata identity request failed") + } + request.Header.Set("Metadata-Flavor", "Google") + response, err := provider.client.Do(request) + if err != nil { + return "", errors.New("metadata identity request failed") + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return "", fmt.Errorf("metadata identity request returned status %d", response.StatusCode) + } + tokenBytes, err := io.ReadAll(io.LimitReader(response.Body, maxIdentityTokenBody+1)) + if err != nil { + return "", errors.New("read metadata identity response failed") + } + if len(tokenBytes) == 0 || len(tokenBytes) > maxIdentityTokenBody { + return "", errors.New("metadata identity response has an invalid size") + } + token := strings.TrimSpace(string(tokenBytes)) + if token == "" || strings.ContainsAny(token, " \t\r\n") { + return "", errors.New("metadata identity response is not a token") + } + expiresAt, err := identityTokenExpiry(token) + if err != nil { + return "", err + } + if time.Until(expiresAt) <= 0 { + return "", errors.New("metadata identity token is already expired") + } + provider.token = token + provider.expiresAt = expiresAt + return token, nil +} + +func identityTokenExpiry(token string) (time.Time, error) { + parts := strings.Split(token, ".") + if len(parts) != 3 { + return time.Time{}, errors.New("metadata identity response is not a JWT") + } + payload, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + return time.Time{}, errors.New("metadata identity JWT payload is invalid") + } + var claims struct { + ExpiresAt json.Number `json:"exp"` + } + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.UseNumber() + if err := decoder.Decode(&claims); err != nil { + return time.Time{}, errors.New("metadata identity JWT claims are invalid") + } + expiry, err := claims.ExpiresAt.Int64() + if err != nil || expiry <= 0 { + return time.Time{}, errors.New("metadata identity JWT expiration is invalid") + } + return time.Unix(expiry, 0), nil +} + +func (transport *identityTokenTransport) RoundTrip(request *http.Request) (*http.Response, error) { + token, err := transport.provider.Token(request.Context(), transport.audience) + if err != nil { + return nil, fmt.Errorf("authorize Vault upstream: %w", err) + } + outbound := request.Clone(request.Context()) + outbound.Header = request.Header.Clone() + outbound.Header.Set("X-Serverless-Authorization", "Bearer "+token) + base := transport.base + if base == nil { + base = http.DefaultTransport + } + return base.RoundTrip(outbound) +} + func createProxyHandler(config *Config) http.Handler { validator := googleTokenValidator{ client: &http.Client{Timeout: 10 * time.Second}, @@ -274,7 +419,37 @@ func createProxyHandler(config *Config) http.Handler { } func createProxyHandlerWithValidator(config *Config, validator tokenValidator) http.Handler { + identityProvider := &metadataIdentityTokenProvider{ + client: newMetadataIdentityHTTPClient(), + endpoint: defaultMetadataIdentityURL, + } + return createProxyHandlerWithDependencies(config, validator, identityProvider) +} + +func newMetadataIdentityHTTPClient() *http.Client { + return &http.Client{ + Timeout: 5 * time.Second, + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + // The metadata endpoint is necessarily link-local plain HTTP. Never let + // environment proxy settings observe the returned service identity token. + Transport: &http.Transport{ + Proxy: nil, + DisableKeepAlives: true, + }, + } +} + +func createProxyHandlerWithDependencies(config *Config, validator tokenValidator, identityProvider identityTokenProvider) http.Handler { proxy := &httputil.ReverseProxy{} + if config.VaultAudience != "" { + proxy.Transport = &identityTokenTransport{ + base: http.DefaultTransport, + provider: identityProvider, + audience: config.VaultAudience, + } + } proxy.Rewrite = func(request *httputil.ProxyRequest) { // Rewrite mode removes the standard client-provided Forwarded and // X-Forwarded-* values before this function runs. Remove extensions as @@ -282,7 +457,7 @@ func createProxyHandlerWithValidator(config *Config, validator tokenValidator) h // audit decisions using an attacker-supplied client IP, host, or scheme. for header := range request.Out.Header { normalized := strings.ToLower(header) - if normalized == "forwarded" || normalized == "x-real-ip" || strings.HasPrefix(normalized, "x-forwarded-") { + if normalized == "forwarded" || normalized == "x-real-ip" || normalized == "x-serverless-authorization" || strings.HasPrefix(normalized, "x-forwarded-") { request.Out.Header.Del(header) } } diff --git a/main_test.go b/main_test.go index ab17f8c..0a575f9 100644 --- a/main_test.go +++ b/main_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -14,6 +15,7 @@ import ( "path/filepath" "strings" "testing" + "time" ) type tokenValidatorFunc func(context.Context, string, map[string]struct{}) error @@ -28,6 +30,12 @@ func (fn roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) return fn(request) } +type identityTokenProviderFunc func(context.Context, string) (string, error) + +func (fn identityTokenProviderFunc) Token(ctx context.Context, audience string) (string, error) { + return fn(ctx, audience) +} + func TestIsPublicRoute(t *testing.T) { routes := []string{ "/.well-known/**", @@ -112,10 +120,22 @@ admin_emails: [admin@example.com] wantPort: 8443, wantAdmins: 1, }, + { + name: "private upstream audience", + yaml: `vault_addr: https://vault-runtime.example.run.app +vault_audience: https://vault-runtime.example.run.app +admin_emails: [admin@example.com] +`, + wantPort: 8080, + wantAdmins: 1, + }, {name: "unknown field", yaml: "vault_addr: http://vault:8200\nadmin_emails: [admin@example.com]\nsurprise: true\n", wantErr: "field surprise not found"}, {name: "relative URL", yaml: "vault_addr: vault:8200\nadmin_emails: [admin@example.com]\n", wantErr: "must use http or https"}, {name: "URL credentials", yaml: "vault_addr: https://user:pass@vault.example.com\nadmin_emails: [admin@example.com]\n", wantErr: "must not contain credentials"}, {name: "URL path", yaml: "vault_addr: https://vault.example.com/v1\nadmin_emails: [admin@example.com]\n", wantErr: "must not contain a path"}, + {name: "audience with plaintext upstream", yaml: "vault_addr: http://vault:8200\nvault_audience: https://vault.example.run.app\nadmin_emails: [admin@example.com]\n", wantErr: "must use HTTPS"}, + {name: "audience path", yaml: "vault_addr: https://vault.example.run.app\nvault_audience: https://vault.example.run.app/path\nadmin_emails: [admin@example.com]\n", wantErr: "must not contain credentials, a path"}, + {name: "audience credentials", yaml: "vault_addr: https://vault.example.run.app\nvault_audience: https://user@example.run.app\nadmin_emails: [admin@example.com]\n", wantErr: "must not contain credentials"}, {name: "empty admins", yaml: "vault_addr: http://vault:8200\nadmin_emails: [' ']\n", wantErr: "invalid admin email"}, {name: "invalid email", yaml: "vault_addr: http://vault:8200\nadmin_emails: [not-an-email]\n", wantErr: "invalid admin email"}, {name: "invalid low port", yaml: "vault_addr: http://vault:8200\nport: -1\nadmin_emails: [admin@example.com]\n", wantErr: "between 1 and 65535"}, @@ -150,6 +170,97 @@ admin_emails: [admin@example.com] } } +func TestMetadataIdentityTokenProviderCachesValidToken(t *testing.T) { + expires := time.Now().Add(time.Hour).Unix() + payload := base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf(`{"exp":%d}`, expires))) + token := "header." + payload + ".signature" + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + calls++ + if request.Header.Get("Metadata-Flavor") != "Google" { + t.Error("metadata request omitted Metadata-Flavor") + } + if request.URL.Query().Get("audience") != "https://vault-runtime.example.run.app" || request.URL.Query().Get("format") != "full" { + t.Errorf("metadata query = %q", request.URL.RawQuery) + } + _, _ = w.Write([]byte(token)) + })) + defer server.Close() + + provider := &metadataIdentityTokenProvider{client: server.Client(), endpoint: server.URL} + for range 2 { + got, err := provider.Token(context.Background(), "https://vault-runtime.example.run.app") + if err != nil { + t.Fatal(err) + } + if got != token { + t.Fatalf("token = %q, want test token", got) + } + } + if calls != 1 { + t.Fatalf("metadata calls = %d, want one cached fetch", calls) + } +} + +func TestMetadataIdentityClientRejectsProxyAndRedirects(t *testing.T) { + client := newMetadataIdentityHTTPClient() + transport, ok := client.Transport.(*http.Transport) + if !ok { + t.Fatalf("transport = %T", client.Transport) + } + if transport.Proxy != nil || !transport.DisableKeepAlives { + t.Fatal("metadata identity client permits a proxy or reusable plaintext connection") + } + if client.CheckRedirect == nil || !errors.Is(client.CheckRedirect(nil, nil), http.ErrUseLastResponse) { + t.Fatal("metadata identity client permits redirects") + } +} + +func TestIdentityTokenExpiryRejectsMalformedTokens(t *testing.T) { + for _, token := range []string{"", "one.two", "one.%%%25.three", "one.e30.three"} { + if _, err := identityTokenExpiry(token); err == nil { + t.Errorf("identityTokenExpiry(%q) accepted malformed token", token) + } + } +} + +func TestProxyAddsUpstreamIdentityWithoutReplacingVaultAuthorization(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if got := request.Header.Get("X-Serverless-Authorization"); got != "Bearer runtime-id-token" { + t.Errorf("X-Serverless-Authorization = %q", got) + } + if got := request.Header.Get("Authorization"); got != "Bearer vault-client-token" { + t.Errorf("Authorization = %q", got) + } + w.WriteHeader(http.StatusNoContent) + })) + defer upstream.Close() + + config := testConfig(t, upstream.URL, []string{"/v1/sys/health"}) + config.VaultAudience = "https://vault-runtime.example.run.app" + handler := createProxyHandlerWithDependencies( + config, + tokenValidatorFunc(func(context.Context, string, map[string]struct{}) error { + t.Fatal("validator called for public route") + return nil + }), + identityTokenProviderFunc(func(_ context.Context, audience string) (string, error) { + if audience != config.VaultAudience { + t.Fatalf("audience = %q", audience) + } + return "runtime-id-token", nil + }), + ) + request := httptest.NewRequest(http.MethodGet, "/v1/sys/health", nil) + request.Header.Set("Authorization", "Bearer vault-client-token") + request.Header.Set("X-Serverless-Authorization", "Bearer attacker-token") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusNoContent { + t.Fatalf("status = %d, want %d", response.Code, http.StatusNoContent) + } +} + func TestLoadConfigEnvironmentTakesPrecedence(t *testing.T) { t.Setenv("VAULT_PROXY_YAML", "vault_addr: http://env-vault:8200\nadmin_emails: [admin@example.com]\n") config, err := loadConfig(filepath.Join(t.TempDir(), "missing.yaml")) @@ -356,6 +467,7 @@ func TestProxyHandlerRemovesSpoofedForwardingHeaders(t *testing.T) { "X-Forwarded-Port", "X-Forwarded-Proto", "X-Real-IP", + "X-Serverless-Authorization", } { if values := request.Header.Values(header); len(values) != 0 { t.Errorf("upstream received spoofable %s header %q", header, values) @@ -379,6 +491,7 @@ func TestProxyHandlerRemovesSpoofedForwardingHeaders(t *testing.T) { request.Header.Set("X-Forwarded-Port", "443") request.Header.Set("X-Forwarded-Proto", "https") request.Header.Set("X-Real-IP", "198.51.100.10") + request.Header.Set("X-Serverless-Authorization", "Bearer attacker-token") response := httptest.NewRecorder() handler.ServeHTTP(response, request)