diff --git a/go.mod b/go.mod index af23cd56..253edf3d 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.9 github.com/spf13/viper v1.10.1 - github.com/stackql-labs/omnisdk v0.1.0-alpha07 + github.com/stackql-labs/omnisdk v0.1.1-alpha06 github.com/stackql/any-sdk v0.5.4-alpha01 github.com/stackql/go-suffix-map v0.0.1-alpha01 github.com/stackql/psql-wire v0.1.2-beta01 diff --git a/go.sum b/go.sum index 02190eb0..dfe50192 100644 --- a/go.sum +++ b/go.sum @@ -348,8 +348,8 @@ github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= -github.com/stackql-labs/omnisdk v0.1.0-alpha07 h1:kGUtupkwAVZWwCkMQZ4vjCD5jIDPmGCSq/OdH4Dmos0= -github.com/stackql-labs/omnisdk v0.1.0-alpha07/go.mod h1:SHUOryRWXeZip4JTWeA8iwmz4lYdBSp0nbWXDX29JrU= +github.com/stackql-labs/omnisdk v0.1.1-alpha06 h1:jVetiuZa6J1uWJ4vPc/YQtol4l7Lgoq281UWvYXw6GM= +github.com/stackql-labs/omnisdk v0.1.1-alpha06/go.mod h1:WzvNj/bVv53yGFsVJpYWCJC1xAEdmQSFJl9eVpkRpCY= github.com/stackql/any-sdk v0.5.4-alpha01 h1:AyjD2Hyk7D8v1fHHtLpfeqQRxKuj7rC71dzi1Zh9LoA= github.com/stackql/any-sdk v0.5.4-alpha01/go.mod h1:BiE8uiAJMUa8n4U/yMlhvrVhE2M+5Rt8utBXYkwj9To= github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8= diff --git a/internal/stackql/cmd/root.go b/internal/stackql/cmd/root.go index a313a336..d233a221 100644 --- a/internal/stackql/cmd/root.go +++ b/internal/stackql/cmd/root.go @@ -155,7 +155,8 @@ func init() { rootCmd.PersistentFlags().StringVar(&runtimeCtx.StoreTxnCfgRaw, dto.StoreTxnCfgRawKey, "{}", "JSON / YAML string representing Txn store config") rootCmd.PersistentFlags().StringVar(&runtimeCtx.GCCfgRaw, dto.GCCfgRawKey, "{}", "JSON / YAML string representing GC config") rootCmd.PersistentFlags().StringVar(&runtimeCtx.ACIDCfgRaw, dto.ACIDCfgRawKey, "{}", "JSON / YAML string representing ACID config") - rootCmd.PersistentFlags().StringVar(&previewCfgRaw, intrinsic.CfgRawKey, "{}", "JSON string configuring the "+intrinsic.ProviderName+" provider backend; keys: batchSize, flushInterval, endpoint") + rootCmd.PersistentFlags().StringVar(&previewCfgRaw, intrinsic.CfgRawKey, "{}", "JSON string configuring the "+intrinsic.ProviderName+ + " provider backend; keys: batchSize, flushInterval, endpoint, unstable") rootCmd.PersistentFlags().StringVar(&runtimeCtx.SessionCtxRaw, dto.SessionCtxKey, "{}", "JSON / YAML string representing session config") rootCmd.PersistentFlags().IntVar(&runtimeCtx.APIRequestTimeout, dto.APIRequestTimeoutKey, 45, "API request timeout in seconds, 0 for no timeout.") //nolint:mnd // TODO: investigate rootCmd.PersistentFlags().StringVar(&dummyString, dto.ColorSchemeKey, "", "DEPRECATED: color schems no longer active") diff --git a/internal/stackql/handler/handler.go b/internal/stackql/handler/handler.go index d9157527..cfb562b9 100644 --- a/internal/stackql/handler/handler.go +++ b/internal/stackql/handler/handler.go @@ -323,6 +323,15 @@ func (hc *standardHandlerContext) GetSupportedProviders(extended bool) (map[stri } else { retVal[pn] = getProviderMap(pn, pd) } + // Every locally available bundle is also addressable document-first, + // once that has been opted into. + if intrinsic.IsUnstableEnabled() { + unstable := intrinsic.UnstablePrefix + pn + retVal[unstable] = map[string]interface{}{ + "name": unstable, + "version": intrinsic.ProviderVersion, + } + } } return retVal, nil } diff --git a/internal/stackql/intrinsic/doc.go b/internal/stackql/intrinsic/doc.go new file mode 100644 index 00000000..2bbb591d --- /dev/null +++ b/internal/stackql/intrinsic/doc.go @@ -0,0 +1,266 @@ +package intrinsic + +import ( + "context" + "fmt" + "path/filepath" + "sort" + "strings" + + "github.com/stackql-labs/omnisdk/pkg/docparse/aot" + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql/any-sdk/pkg/dto" + "github.com/stackql/any-sdk/public/formulation" + "github.com/stackql/psql-wire/pkg/sqldata" + "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/util" + + "github.com/stackql/stackql-parser/go/vt/sqlparser" + "gopkg.in/yaml.v2" +) + +// UnstablePrefix names the document-driven providers. The convention is +// omnisdk's own, and its addresses carry the prefixed provider name, so this is +// its constant rather than a copy of the literal. +const UnstablePrefix = aot.DefaultProviderPrefix + +// IsUnstableEnabled reports whether the document-driven providers were opted +// into. They are documents read straight from disk, with none of the registry's +// curation behind them, so nothing exposes them until a caller asks. +func IsUnstableEnabled() bool { + return previewCfg.getUnstableEnabled() +} + +// docProvider is the bundle behind an unstable provider name, or false. +func docProvider(name string) (string, bool) { + if !IsUnstableEnabled() { + return "", false + } + trimmed := strings.TrimSpace(name) + if !strings.HasPrefix(strings.ToLower(trimmed), UnstablePrefix) { + return "", false + } + bundle := trimmed[len(UnstablePrefix):] + if bundle == "" { + return "", false + } + return bundle, true +} + +// bundleAliases maps the provider name stackql presents onto the directory the +// registry actually wrote. Keeping them apart matters: the name a caller typed +// is the one echoed back, and it is the only one they can address. +var bundleAliases = map[string]string{ //nolint:gochecknoglobals // fixed mapping + "google": "googleapis.com", +} + +func bundleDir(label string) string { + if dir, ok := bundleAliases[strings.ToLower(label)]; ok { + return dir + } + return label +} + +// localDocRoot is where provider documents are read from, resolved exactly as +// the canonical providers resolve it, so an unstable provider reads whatever +// documents are already on disk rather than needing assets of its own. +func localDocRoot(runtimeCtx dto.RuntimeCtx) string { + var registryCfg formulation.RegistryConfig + if err := yaml.Unmarshal([]byte(runtimeCtx.RegistryRaw), ®istryCfg); err == nil { + if registryCfg.LocalDocRoot != "" { + return registryCfg.LocalDocRoot + } + if strings.HasPrefix(registryCfg.RegistryURL, "file:") { + return filepath.Clean( + filepath.Join(strings.TrimPrefix(registryCfg.RegistryURL, "file:"), "..")) + } + } + return runtimeCtx.ApplicationFilesRootPath +} + +// docRoot is the bundle's own directory inside stackql's document root. The +// versioned directory is used rather than the root itself, because a registry +// root is addressed as ".." and a provider whose +// name carries a dot ("googleapis.com") cannot be named that way. +func docRoot(ctx queryContext, bundle string) (string, error) { + root := filepath.Join(localDocRoot(ctx.GetRuntimeContext()), "src", bundleDir(bundle)) + matches, err := filepath.Glob(filepath.Join(root, "*", "provider.yaml")) + if err != nil || len(matches) == 0 { + return "", fmt.Errorf("no provider document for '%s%s' under '%s'", UnstablePrefix, bundle, root) + } + sort.Strings(matches) + return filepath.Dir(matches[len(matches)-1]), nil +} + +// docServices lists the services a bundle ships documents for. +func docServices(ctx queryContext, bundle string) ([]string, error) { + dir, err := docRoot(ctx, bundle) + if err != nil { + return nil, err + } + services, _, err := omnisdk.DocCatalog(dir, bundle) + return services, err +} + +// docResourceTables presents a service's resources as relations. +func docResourceTables(ctx queryContext, bundle, service string) ([]table, error) { + dir, err := docRoot(ctx, bundle) + if err != nil { + return nil, err + } + resources, err := omnisdk.DocResources(dir, bundle, service) + if err != nil { + return nil, err + } + sort.Strings(resources) + out := make([]table, 0, len(resources)) + for _, resource := range resources { + out = append(out, table{service: service, name: resource, isData: true}) + } + return out, nil +} + +// docMethods lists a resource's methods as the document declares them. +func docMethods(ctx queryContext, bundle, service, resource string) ([]relationMethod, error) { + dir, err := docRoot(ctx, bundle) + if err != nil { + return nil, err + } + methods, err := omnisdk.DocMethods(dir, bundle, service, resource) + if err != nil { + return nil, err + } + out := make([]relationMethod, 0, len(methods)) + for _, method := range methods { + out = append(out, relationMethod{name: method.Name, description: method.OperationID}) + } + return out, nil +} + +// docSelectFunc routes a SELECT over a document-driven relation. The address is +// the bundle's own "..", and the plan it yields +// streams exactly as a hand-authored one does. +func docSelectFunc( + ctx queryContext, + node *sqlparser.Select, + bundle, service, resource string, +) (func() internaldto.ExecutorOutput, bool) { + if unsupported := unsupportedClauses(node); len(unsupported) > 0 { + return refuse(fmt.Errorf( + "relation '%s%s.%s.%s' streams its rows, so %s cannot be applied; remove %s from the query", + UnstablePrefix, bundle, service, resource, + strings.Join(unsupported, ", "), pluralClause(len(unsupported)))), true + } + params, badPredicates := equalityPredicates(node.Where) + if len(badPredicates) > 0 { + return refuse(fmt.Errorf( + "relation '%s%s.%s.%s' streams its rows, so only equality predicates are applied; "+ + "%s cannot be honoured", + UnstablePrefix, bundle, service, resource, strings.Join(badPredicates, ", "))), true + } + address := fmt.Sprintf("%s%s.%s.%s", UnstablePrefix, bundle, service, resource) + return func() internaldto.ExecutorOutput { + input := previewCfg + dir, dirErr := docRoot(ctx, bundle) + if dirErr != nil { + return internaldto.NewErroneousExecutorOutput(dirErr) + } + plan, err := omnisdk.NewFromCatalog(dir, address, omnisdk.Args{ + Params: params, + Auth: omnisdkAuth(providerAuthContext(ctx, bundle)), + Endpoint: input.getEndpoint(), + InsecureSkipTLSVerify: input.getInsecureSkipTLSVerify(), + }) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + rows, openErr := plan.Open(context.Background()) + if openErr != nil { + return internaldto.NewErroneousExecutorOutput(openErr) + } + // A document declares no egress schema, so the columns are those the + // first row carries; projection is applied over them. + stream := &rowStream{ + rows: rows, + batchSize: input.getBatchSize(), + flushInterval: input.getFlushInterval(), + table: sqldata.NewSQLTable(0, resource), + typCfg: ctx.GetTypingConfig(), + projection: node.SelectExprs, + } + primed, readErr := newPrimedStream(stream) + if readErr != nil { + return internaldto.NewErroneousExecutorOutput(readErr) + } + return internaldto.NewExecutorOutput(primed, nil, nil, nil, nil) + }, true +} + +func refuse(err error) func() internaldto.ExecutorOutput { + return func() internaldto.ExecutorOutput { + return internaldto.NewErroneousExecutorOutput(err) + } +} + +func showDocServices(ctx queryContext, bundle string, extended bool) internaldto.ExecutorOutput { + services, err := docServices(ctx, bundle) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + rows := make(map[string]map[string]interface{}, len(services)) + for i, service := range services { + row := map[string]interface{}{"id": service, "name": service, "title": service} + if extended { + row["description"] = service + row["version"] = ProviderVersion + row["preferred"] = nil + } + rows[fmt.Sprintf("%06d", i)] = row + } + return prepare(ctx, formulation.GetServicesHeader(extended), rows, util.DefaultRowSort) +} + +func showDocResources( + ctx queryContext, bundle, service string, extended bool) internaldto.ExecutorOutput { + tables, err := docResourceTables(ctx, bundle, service) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + rows := make(map[string]map[string]interface{}, len(tables)) + for i, tbl := range tables { + row := map[string]interface{}{ + "id": fmt.Sprintf("%s%s.%s.%s", UnstablePrefix, bundle, service, tbl.name), + "name": tbl.name, + } + if extended { + row["description"] = tbl.description + } + rows[fmt.Sprintf("%06d", i)] = row + } + return prepare(ctx, formulation.GetResourcesHeader(extended), rows, util.DefaultRowSort) +} + +func showDocMethods( + ctx queryContext, bundle, service, resource string, extended bool) internaldto.ExecutorOutput { + methods, err := docMethods(ctx, bundle, service, resource) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + columnOrder := []string{"MethodName", "RequiredParams", "SQLVerb"} + if extended { + columnOrder = append(columnOrder, "description") + } + rows := make(map[string]map[string]interface{}, len(methods)) + for i, method := range methods { + row := map[string]interface{}{ + "MethodName": method.name, + "RequiredParams": strings.Join(method.requiredParams, ", "), + "SQLVerb": strings.ToUpper(selectMethodName), + } + if extended { + row["description"] = method.description + } + rows[fmt.Sprintf("%06d", i)] = row + } + return prepare(ctx, columnOrder, rows, util.DefaultRowSort) +} diff --git a/internal/stackql/intrinsic/intrinsic.go b/internal/stackql/intrinsic/intrinsic.go index 213408b6..72a2daf7 100644 --- a/internal/stackql/intrinsic/intrinsic.go +++ b/internal/stackql/intrinsic/intrinsic.go @@ -35,6 +35,7 @@ type column struct { type table struct { service string name string + isData bool description string columns []column } @@ -48,6 +49,7 @@ type queryContext interface { SetCurrentProvider(string) GetTypingConfig() typing.Config GetAuthContext(providerName string) (*dto.AuthCtx, error) + GetRuntimeContext() dto.RuntimeCtx } func GeneratePrimitiveFunc( @@ -76,7 +78,11 @@ func GenerateStreamFunc( } func IsProvider(name string) bool { - return strings.EqualFold(strings.TrimSpace(name), ProviderName) + if strings.EqualFold(strings.TrimSpace(name), ProviderName) { + return true + } + _, isDoc := docProvider(name) + return isDoc } func resolveProvider(providerName string, currentProvider string) string { @@ -132,17 +138,34 @@ func showFunc( extended := isExtended(node.Extended) switch strings.ToUpper(strings.TrimSpace(node.Type)) { case "SERVICES": - if !IsProvider(resolveProvider(node.OnTable.Name.GetRawVal(), currentProvider)) { + provider := resolveProvider(node.OnTable.Name.GetRawVal(), currentProvider) + if bundle, isDoc := docProvider(provider); isDoc { + return func() internaldto.ExecutorOutput { return showDocServices(ctx, bundle, extended) }, true + } + if !IsProvider(provider) { return nil, false } return func() internaldto.ExecutorOutput { return showServices(ctx, extended) }, true case "RESOURCES": serviceStr := node.OnTable.Name.GetRawVal() + if bundle, isDoc := docProvider(resolveProvider(node.OnTable.Qualifier.GetRawVal(), currentProvider)); isDoc { + return func() internaldto.ExecutorOutput { + return showDocResources(ctx, bundle, serviceStr, extended) + }, true + } if !isService(node.OnTable.Qualifier.GetRawVal(), serviceStr, currentProvider) { return nil, false } return func() internaldto.ExecutorOutput { return showResources(ctx, serviceStr, extended) }, true case "METHODS": + if bundle, isDoc := docProvider( + resolveProvider(node.OnTable.QualifierSecond.GetRawVal(), currentProvider)); isDoc { + service := node.OnTable.Qualifier.GetRawVal() + resource := node.OnTable.Name.GetRawVal() + return func() internaldto.ExecutorOutput { + return showDocMethods(ctx, bundle, service, resource, extended) + }, true + } tbl, ok := lookupTable( node.OnTable.QualifierSecond.GetRawVal(), node.OnTable.Qualifier.GetRawVal(), diff --git a/internal/stackql/intrinsic/omnisdk.go b/internal/stackql/intrinsic/omnisdk.go index 2b5e4bb6..95696e09 100644 --- a/internal/stackql/intrinsic/omnisdk.go +++ b/internal/stackql/intrinsic/omnisdk.go @@ -156,9 +156,10 @@ func openStream( } input := previewCfg args := omnisdk.Args{ - Params: params, - Auth: omnisdkAuth(providerAuthContext(ctx, resourcePath)), - Endpoint: input.getEndpoint(), + Params: params, + Auth: omnisdkAuth(providerAuthContext(ctx, resourcePath)), + Endpoint: input.getEndpoint(), + InsecureSkipTLSVerify: input.getInsecureSkipTLSVerify(), } plan, err := omnisdk.Default().New(method.Path, args) if err != nil { @@ -188,6 +189,7 @@ type rowStream struct { producerOnce sync.Once columns []column table sqldata.ISQLTable + projection sqlparser.SelectExprs typCfg columnFactory done bool } @@ -263,6 +265,11 @@ func (rs *rowStream) result(batch []omnisdk.Row) sqldata.ISQLResult { for _, name := range sortedKeys(batch[0]) { rs.columns = append(rs.columns, column{name: name}) } + if len(rs.projection) > 0 { + if selected, err := projection(rs.projection, rs.columns); err == nil { + rs.columns = selected + } + } } columns := make([]sqldata.ISQLColumn, 0, len(rs.columns)) for _, col := range rs.columns { @@ -312,6 +319,11 @@ func selectFunc( if !ok { return nil, false } + if bundle, isDoc := docProvider( + resolveProvider(tableName.QualifierSecond.GetRawVal(), currentProvider)); isDoc { + return docSelectFunc(ctx, node, bundle, + tableName.Qualifier.GetRawVal(), tableName.Name.GetRawVal()) + } if !strings.EqualFold(tableName.Qualifier.GetRawVal(), auditService) || !IsProvider(resolveProvider(tableName.QualifierSecond.GetRawVal(), currentProvider)) { return nil, false @@ -644,12 +656,16 @@ type backendInput interface { getBatchSize() int getEndpoint() string getFlushInterval() time.Duration + getInsecureSkipTLSVerify() bool + getUnstableEnabled() bool } type standardBackendInput struct { - batchSize int - endpoint string - flushInterval time.Duration + batchSize int + endpoint string + flushInterval time.Duration + insecureSkipTLSVerify bool + unstableEnabled bool } // previewCfg is the parsed --preview argument. Cobra binds the raw string in @@ -665,9 +681,11 @@ const CfgRawKey = "preview" // either form omnisdk does: a base URL for every service, or an object of // service to override. Both ride through as the string omnisdk parses. type previewCfgDTO struct { - BatchSize int `json:"batchSize"` - FlushInterval string `json:"flushInterval"` - Endpoint json.RawMessage `json:"endpoint"` + BatchSize int `json:"batchSize"` + FlushInterval string `json:"flushInterval"` + Endpoint json.RawMessage `json:"endpoint"` + InsecureSkipTLSVerify bool `json:"insecureSkipTLSVerify"` + Unstable bool `json:"unstable"` } func (c previewCfgDTO) endpoint() string { @@ -694,9 +712,11 @@ func Init(raw string) { func newBackendInput(cfg previewCfgDTO) backendInput { rv := &standardBackendInput{ - batchSize: defaultBatchSize, - endpoint: cfg.endpoint(), - flushInterval: defaultFlushInterval, + batchSize: defaultBatchSize, + endpoint: cfg.endpoint(), + flushInterval: defaultFlushInterval, + insecureSkipTLSVerify: cfg.InsecureSkipTLSVerify, + unstableEnabled: cfg.Unstable, } if cfg.BatchSize > 0 { rv.batchSize = cfg.BatchSize @@ -713,6 +733,10 @@ func (b *standardBackendInput) getEndpoint() string { return b.endpoint } func (b *standardBackendInput) getFlushInterval() time.Duration { return b.flushInterval } +func (b *standardBackendInput) getInsecureSkipTLSVerify() bool { return b.insecureSkipTLSVerify } + +func (b *standardBackendInput) getUnstableEnabled() bool { return b.unstableEnabled } + // sourceKey is the row key a column reads from: its own name, unless an alias // renamed it. func (c column) sourceKey() string { diff --git a/test/assets/expected/preview/omni-iam-principals.jsonl b/test/assets/expected/preview/omni-iam-principals.jsonl new file mode 100644 index 00000000..8fb13465 --- /dev/null +++ b/test/assets/expected/preview/omni-iam-principals.jsonl @@ -0,0 +1,13 @@ +{"created": "2024-01-05T00:00:00Z", "enabled": "true", "google_org": "123456789", "google_project": null, "grant": "Global Reader", "mfa": null, "principal": "reviewer@example.invalid", "principal_id": "00000000-0000-0000-0000-000000000001", "principal_type": "user", "provider": "entra", "region": "us-east-1", "scope": "mock-tenant"} +{"created": "2024-01-05T00:00:00Z", "enabled": null, "google_org": "123456789", "google_project": null, "grant": "ReadOnlyAccess", "mfa": "true", "principal": "audit-reviewer", "principal_id": "AIDAEXAMPLEREVIEWER01", "principal_type": "user", "provider": "aws", "region": "us-east-1", "scope": "000000000000"} +{"created": "2024-02-11T00:00:00Z", "enabled": null, "google_org": "123456789", "google_project": null, "grant": "AmazonS3FullAccess", "mfa": "false", "principal": "ci-deployer", "principal_id": "AIDAEXAMPLEDEPLOYER02", "principal_type": "user", "provider": "aws", "region": "us-east-1", "scope": "000000000000"} +{"created": "2024-02-11T00:00:00Z", "enabled": null, "google_org": "123456789", "google_project": null, "grant": "IAMReadOnlyAccess", "mfa": "false", "principal": "ci-deployer", "principal_id": "AIDAEXAMPLEDEPLOYER02", "principal_type": "user", "provider": "aws", "region": "us-east-1", "scope": "000000000000"} +{"created": "2024-03-19T00:00:00Z", "enabled": null, "google_org": "123456789", "google_project": null, "grant": null, "mfa": "false", "principal": "no-grants-user", "principal_id": "AIDAEXAMPLENOGRANTS03", "principal_type": "user", "provider": "aws", "region": "us-east-1", "scope": "000000000000"} +{"created": "2024-04-02T00:00:00Z", "enabled": "false", "google_org": "123456789", "google_project": null, "grant": "Application Administrator", "mfa": null, "principal": "operator@example.invalid", "principal_id": "00000000-0000-0000-0000-000000000002", "principal_type": "user", "provider": "entra", "region": "us-east-1", "scope": "mock-tenant"} +{"created": "2024-04-02T00:00:00Z", "enabled": "false", "google_org": "123456789", "google_project": null, "grant": "Directory Readers", "mfa": null, "principal": "operator@example.invalid", "principal_id": "00000000-0000-0000-0000-000000000002", "principal_type": "user", "provider": "entra", "region": "us-east-1", "scope": "mock-tenant"} +{"created": null, "enabled": null, "google_org": "123456789", "google_project": null, "grant": "roles/editor", "mfa": null, "principal": "ci@example-project.iam.gserviceaccount.com", "principal_id": "serviceAccount:ci@example-project.iam.gserviceaccount.com", "principal_type": "service_principal", "provider": "gcp", "region": "us-east-1", "scope": "proj-100"} +{"created": null, "enabled": null, "google_org": "123456789", "google_project": null, "grant": "roles/editor", "mfa": null, "principal": "ci@example-project.iam.gserviceaccount.com", "principal_id": "serviceAccount:ci@example-project.iam.gserviceaccount.com", "principal_type": "service_principal", "provider": "gcp", "region": "us-east-1", "scope": "proj-root"} +{"created": null, "enabled": null, "google_org": "123456789", "google_project": null, "grant": "roles/editor", "mfa": null, "principal": "operator@example.invalid", "principal_id": "user:operator@example.invalid", "principal_type": "user", "provider": "gcp", "region": "us-east-1", "scope": "proj-100"} +{"created": null, "enabled": null, "google_org": "123456789", "google_project": null, "grant": "roles/editor", "mfa": null, "principal": "operator@example.invalid", "principal_id": "user:operator@example.invalid", "principal_type": "user", "provider": "gcp", "region": "us-east-1", "scope": "proj-root"} +{"created": null, "enabled": null, "google_org": "123456789", "google_project": null, "grant": "roles/viewer", "mfa": null, "principal": "reviewer@example.invalid", "principal_id": "user:reviewer@example.invalid", "principal_type": "user", "provider": "gcp", "region": "us-east-1", "scope": "proj-100"} +{"created": null, "enabled": null, "google_org": "123456789", "google_project": null, "grant": "roles/viewer", "mfa": null, "principal": "reviewer@example.invalid", "principal_id": "user:reviewer@example.invalid", "principal_type": "user", "provider": "gcp", "region": "us-east-1", "scope": "proj-root"} diff --git a/test/python/stackql_test_tooling/StackQLInterfaces.py b/test/python/stackql_test_tooling/StackQLInterfaces.py index 9eeffb79..2dd8b0b2 100644 --- a/test/python/stackql_test_tooling/StackQLInterfaces.py +++ b/test/python/stackql_test_tooling/StackQLInterfaces.py @@ -311,7 +311,7 @@ def _run_stackql_exec_command_docker( invocation_str = f"{sleep_prefix}stackql exec {' '.join(supplied_args)} '{query_escaped}'" if query_from_input_file_path: invocation_str = f"{sleep_prefix}stackql exec {' '.join(supplied_args)}" - res = super().run_process( + docker_args = ( "docker", "compose", "-p", @@ -323,6 +323,13 @@ def _run_stackql_exec_command_docker( "bash", "-c", invocation_str, + ) + if cfg.pop('start_only', False): + # Non-blocking, so the caller can watch stdout grow. Robot's Process + # library is used from the main thread; no threads are involved. + return super().start_process(*docker_args, **cfg) + res = super().run_process( + *docker_args, **cfg ) self.log(res.stdout) @@ -511,6 +518,14 @@ def _run_stackql_exec_command_native( supplied_args.append(f"--execution.concurrency.limit={self._concurrency_limit}") if not query_from_input_file_path: supplied_args.append(query) + if cfg.pop('start_only', False): + # Non-blocking, so the caller can watch stdout grow. Robot's Process + # library is used from the main thread; no threads are involved. + return super().start_process( + *supplied_args, + *args, + **cfg + ) res = super().run_process( *supplied_args, *args, @@ -950,39 +965,33 @@ def should_stackql_exec_stream_incrementally( """ Assert rows reach the output stream while the query is still running. - The query is run with stdout redirected to a file, and that file is sampled - while the process is alive. A run that buffers its rows writes the file in - one go at the end, so the samples show a single jump; a run that streams - shows the file growing at several distinct times. + The query is started without blocking and its stdout file is sampled from + this thread while the process is alive. A run that buffers its rows writes + the file once at the end, so the samples show a single jump; a run that + streams shows it growing at several distinct times. """ - import threading stdout_path = cfg.get('stdout') if not stdout_path: raise Exception('should_stackql_exec_stream_incrementally requires stdout=') if os.path.exists(stdout_path): os.remove(stdout_path) - result_holder = {} - - def _run(): - try: - result_holder['result'] = self._run_stackql_exec_command( - stackql_exe, okta_secret_str, github_secret_str, k8s_secret_str, - registry_cfg, auth_cfg_str, sql_backend_cfg_str, query, *args, **cfg - ) - except Exception as exc: - result_holder['error'] = exc - - worker = threading.Thread(target=_run) - worker.start() + timeout = float(cfg.pop('stream_timeout_seconds', 120)) + handle = self._run_stackql_exec_command( + stackql_exe, okta_secret_str, github_secret_str, k8s_secret_str, + registry_cfg, auth_cfg_str, sql_backend_cfg_str, query, *args, + start_only=True, **cfg + ) samples = [] - while worker.is_alive(): + deadline = time.time() + timeout + while self.is_process_running(handle): + if time.time() > deadline: + self.terminate_process(handle) + raise Exception('query did not complete within the streaming timeout') size = os.path.getsize(stdout_path) if os.path.exists(stdout_path) else 0 if not samples or size != samples[-1][1]: samples.append((time.time(), size)) time.sleep(0.02) - worker.join() - if 'error' in result_holder: - raise result_holder['error'] + self.wait_for_process(handle, timeout=timeout) growth = [s for s in samples if s[1] > 0] with open(stdout_path) as fh: diff --git a/test/python/stackql_test_tooling/flask/omnisdk/app.py b/test/python/stackql_test_tooling/flask/omnisdk/app.py index 8ace9b15..92f4361f 100644 --- a/test/python/stackql_test_tooling/flask/omnisdk/app.py +++ b/test/python/stackql_test_tooling/flask/omnisdk/app.py @@ -28,6 +28,9 @@ app = Flask(__name__, template_folder=str(HERE / "templates")) BUCKETS = json.loads((COLL / "buckets.json").read_text()) PROVISION = json.loads((COLL / "ec2" / "provision.json").read_text()) +IAM_USERS = json.loads((COLL / "iam_users.json").read_text()) +ENTRA_USERS = json.loads((COLL / "entra_users.json").read_text()) +GCP_IAM_POLICY = json.loads((COLL / "gcp_iam_policy.json").read_text()) NOT_FOUND = ( '' @@ -64,6 +67,8 @@ def list_buckets(): def ec2_query(): # EC2 Query API: params ride the POST form body (Action, VpcId, ...). action = request.form.get("Action") + if action in ("ListUsers", "ListAttachedUserPolicies", "ListMFADevices"): + return _iam_query(action) if action == "CreateVpc": return Response(render_template("create_vpc.xml.j2", **PROVISION), mimetype=XML) if action == "CreateSubnet": @@ -74,6 +79,63 @@ def ec2_query(): return Response("InvalidAction", status=400, mimetype=XML) +def _iam_query(action): + # AWS IAM Query API. Left-outer by nature: a user with no policy or no MFA + # device still answers, with an empty member list. + if action == "ListUsers": + members = "".join( + f"{u['name']}{u['id']}" + f"{u['created']}" + f"arn:aws:iam::000000000000:user/{u['name']}" + for u in IAM_USERS + ) + body = (f"{members}" + "false") + return Response(body, mimetype=XML) + who = request.form.get("UserName") + user = next((u for u in IAM_USERS if u["name"] == who), None) + if action == "ListAttachedUserPolicies": + members = "".join( + f"{p}" + f"arn:aws:iam::aws:policy/{p}" + for p in (user or {}).get("policies", []) + ) + body = ("" + f"{members}" + "") + return Response(body, mimetype=XML) + members = "".join( + f"{m}{who}" + for m in (user or {}).get("mfa", []) + ) + body = ("" + f"{members}" + "") + return Response(body, mimetype=XML) + + +@app.get("/v1.0/users") +def entra_users(): + return jsonify({"value": [ + {k: u[k] for k in ("userPrincipalName", "id", "accountEnabled", "createdDateTime")} + for u in ENTRA_USERS + ]}) + + +@app.get("/v1.0/users//memberOf") +def entra_member_of(principal_id): + user = next((u for u in ENTRA_USERS if u["id"] == principal_id), None) + return jsonify({"value": [ + {"displayName": r, "@odata.type": "#microsoft.graph.directoryRole"} + for r in (user or {}).get("roles", []) + ]}) + + +@app.post("/v3/projects/:getIamPolicy") +def gcp_get_iam_policy(project): + return jsonify(GCP_IAM_POLICY) + + @app.get("/") def bucket_op(bucket): if _DELAY_SECONDS > 0: diff --git a/test/python/stackql_test_tooling/flask/omnisdk/collateral/entra_users.json b/test/python/stackql_test_tooling/flask/omnisdk/collateral/entra_users.json new file mode 100644 index 00000000..a391462d --- /dev/null +++ b/test/python/stackql_test_tooling/flask/omnisdk/collateral/entra_users.json @@ -0,0 +1,21 @@ +[ + { + "userPrincipalName": "reviewer@example.invalid", + "id": "00000000-0000-0000-0000-000000000001", + "accountEnabled": true, + "createdDateTime": "2024-01-05T00:00:00Z", + "roles": [ + "Global Reader" + ] + }, + { + "userPrincipalName": "operator@example.invalid", + "id": "00000000-0000-0000-0000-000000000002", + "accountEnabled": false, + "createdDateTime": "2024-04-02T00:00:00Z", + "roles": [ + "Directory Readers", + "Application Administrator" + ] + } +] \ No newline at end of file diff --git a/test/python/stackql_test_tooling/flask/omnisdk/collateral/gcp_iam_policy.json b/test/python/stackql_test_tooling/flask/omnisdk/collateral/gcp_iam_policy.json new file mode 100644 index 00000000..d0a68f04 --- /dev/null +++ b/test/python/stackql_test_tooling/flask/omnisdk/collateral/gcp_iam_policy.json @@ -0,0 +1,17 @@ +{ + "bindings": [ + { + "role": "roles/viewer", + "members": [ + "user:reviewer@example.invalid" + ] + }, + { + "role": "roles/editor", + "members": [ + "serviceAccount:ci@example-project.iam.gserviceaccount.com", + "user:operator@example.invalid" + ] + } + ] +} \ No newline at end of file diff --git a/test/python/stackql_test_tooling/flask/omnisdk/collateral/iam_users.json b/test/python/stackql_test_tooling/flask/omnisdk/collateral/iam_users.json new file mode 100644 index 00000000..e06887cd --- /dev/null +++ b/test/python/stackql_test_tooling/flask/omnisdk/collateral/iam_users.json @@ -0,0 +1,30 @@ +[ + { + "name": "audit-reviewer", + "id": "AIDAEXAMPLEREVIEWER01", + "created": "2024-01-05T00:00:00Z", + "policies": [ + "ReadOnlyAccess" + ], + "mfa": [ + "arn:aws:iam::000000000000:mfa/audit-reviewer" + ] + }, + { + "name": "ci-deployer", + "id": "AIDAEXAMPLEDEPLOYER02", + "created": "2024-02-11T00:00:00Z", + "policies": [ + "AmazonS3FullAccess", + "IAMReadOnlyAccess" + ], + "mfa": [] + }, + { + "name": "no-grants-user", + "id": "AIDAEXAMPLENOGRANTS03", + "created": "2024-03-19T00:00:00Z", + "policies": [], + "mfa": [] + } +] \ No newline at end of file diff --git a/test/python/stackql_test_tooling/stackql_context.py b/test/python/stackql_test_tooling/stackql_context.py index 9d20c19c..c418e71b 100644 --- a/test/python/stackql_test_tooling/stackql_context.py +++ b/test/python/stackql_test_tooling/stackql_context.py @@ -1146,6 +1146,9 @@ def get_registry_mock_url(execution_env :str) -> str: 'VIEW_SELECT_STAR_AWS_CLOUD_CONTROL_BUCKET_DETAIL_EXPECTED': VIEW_SELECT_STAR_AWS_CLOUD_CONTROL_BUCKET_DETAIL_EXPECTED, 'TEST_TMP_EXEC_APP_ROOT': get_local_temp_path('.exec_app_root.stackql', execution_env), 'TEST_TMP_EXEC_APP_ROOT_NATIVE': get_local_temp_path('.exec_app_root.stackql', 'native'), + # Embedded in an --auth argument, so windows separators must not reach the + # yaml parser as escape sequences. + 'OMNISDK_MOCK_GCP_SA_HOST': get_unix_path(get_local_temp_path('omnisdk-gcp-sa.json', 'native')), } if execution_env == 'docker': rv['AUTH_CFG_STR'] = AUTH_CFG_STR_DOCKER diff --git a/test/robot/functional/stackql.resource b/test/robot/functional/stackql.resource index 5b92840e..0ad34125 100644 --- a/test/robot/functional/stackql.resource +++ b/test/robot/functional/stackql.resource @@ -10,7 +10,6 @@ ${USE_STACKQL_PREINSTALLED} false # to be overridden from command l ${SUNDRY_CONFIG} {} # to be overridden from command line, with string value ${CWD_FOR_EXEC} ${REPOSITORY_ROOT} # works for self repository, can be overwritten when shared ${WEB_SERVICE_LIBRARY} stackql_test_tooling.web_service_keywords -${OMNISDK_MOCK_GCP_SA_HOST} ${REPOSITORY_ROOT}${/}test${/}tmp${/}omnisdk-gcp-sa.json ${OMNISDK_MOCK_URL} http://${LOCAL_HOST_ALIAS}:${MOCKSERVER_PORT_OMNISDK} ${STACKQL_INTERFACE_LIBRARY} stackql_test_tooling.StackQLInterfaces ${CLOUD_INTEGRATION_LIBRARY} stackql_test_tooling.CloudIntegration diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index 26c912b5..ce558f11 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -10622,7 +10622,7 @@ Preview Show Resources Mirrors Omnisdk Catalog ... ${AUTH_CFG_STR} ... ${SQL_BACKEND_CFG_STR_CANONICAL} ... show resources in stackql_preview.audit; - ... name,id\naws_ec2_networks,stackql_preview.audit.aws_ec2_networks\naws_s3_buckets,stackql_preview.audit.aws_s3_buckets\nazure_network_subnets,stackql_preview.audit.azure_network_subnets\nazure_storage_containers,stackql_preview.audit.azure_storage_containers\ngcp_compute_networks,stackql_preview.audit.gcp_compute_networks\ngoogle_storage_buckets,stackql_preview.audit.google_storage_buckets\nomni_storage_buckets,stackql_preview.audit.omni_storage_buckets + ... name,id\naws_ec2_networks,stackql_preview.audit.aws_ec2_networks\naws_iam_principals,stackql_preview.audit.aws_iam_principals\naws_s3_buckets,stackql_preview.audit.aws_s3_buckets\nazure_network_subnets,stackql_preview.audit.azure_network_subnets\nazure_storage_containers,stackql_preview.audit.azure_storage_containers\nentra_identities,stackql_preview.audit.entra_identities\ngcp_compute_networks,stackql_preview.audit.gcp_compute_networks\ngcp_iam_principals,stackql_preview.audit.gcp_iam_principals\ngoogle_storage_buckets,stackql_preview.audit.google_storage_buckets\nomni_iam_principals,stackql_preview.audit.omni_iam_principals\nomni_storage_buckets,stackql_preview.audit.omni_storage_buckets ... \-o\=csv ... stdout=${CURDIR}${/}tmp${/}Preview-Show-Resources-Mirrors-Omnisdk-Catalog.tmp ... stderr=${CURDIR}${/}tmp${/}Preview-Show-Resources-Mirrors-Omnisdk-Catalog-stderr.tmp @@ -10725,12 +10725,16 @@ Preview Jsonl Row Set Is Order Insensitive ... emit, so an order-sensitive comparison would fail here. ${expected} = Catenate SEPARATOR=\n ... {"id":"stackql_preview.audit.omni_storage_buckets","name":"omni_storage_buckets"} - ... {"id":"stackql_preview.audit.aws_s3_buckets","name":"aws_s3_buckets"} - ... {"id":"stackql_preview.audit.gcp_compute_networks","name":"gcp_compute_networks"} - ... {"id":"stackql_preview.audit.aws_ec2_networks","name":"aws_ec2_networks"} + ... {"id":"stackql_preview.audit.omni_iam_principals","name":"omni_iam_principals"} ... {"id":"stackql_preview.audit.google_storage_buckets","name":"google_storage_buckets"} - ... {"id":"stackql_preview.audit.azure_network_subnets","name":"azure_network_subnets"} + ... {"id":"stackql_preview.audit.gcp_iam_principals","name":"gcp_iam_principals"} + ... {"id":"stackql_preview.audit.gcp_compute_networks","name":"gcp_compute_networks"} + ... {"id":"stackql_preview.audit.entra_identities","name":"entra_identities"} ... {"id":"stackql_preview.audit.azure_storage_containers","name":"azure_storage_containers"} + ... {"id":"stackql_preview.audit.azure_network_subnets","name":"azure_network_subnets"} + ... {"id":"stackql_preview.audit.aws_s3_buckets","name":"aws_s3_buckets"} + ... {"id":"stackql_preview.audit.aws_iam_principals","name":"aws_iam_principals"} + ... {"id":"stackql_preview.audit.aws_ec2_networks","name":"aws_ec2_networks"} Should StackQL Exec Inline Jsonl Set Equal ... ${STACKQL_EXE} ... ${OKTA_SECRET_STR} @@ -10779,7 +10783,7 @@ Preview Omni Storage Buckets Jsonl Row Set Matches Expectation ... ${OKTA_SECRET_STR} ... ${GITHUB_SECRET_STR} ... ${K8S_SECRET_STR} - ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${REGISTRY_MOCKED_CFG_STR} ... ${AUTH_CFG_STR} ... ${SQL_BACKEND_CFG_STR_CANONICAL} ... ${query} @@ -10840,3 +10844,119 @@ Preview Order By Is Refused Rather Than Ignored ... streams its rows, so ORDER BY cannot be applied; remove it from the query ... stdout=${CURDIR}${/}tmp${/}Preview-Order-By-Is-Refused-Rather-Than-Ignored.tmp ... stderr=${CURDIR}${/}tmp${/}Preview-Order-By-Is-Refused-Rather-Than-Ignored-stderr.tmp + +Preview Omni Iam Principals Jsonl Row Set Matches Expectation + [Documentation] The cross-cloud access review, order-insensitive. Every + ... identity source the fan-out touches is retargeted at the + ... mock: AWS IAM, the Entra login exchange and Graph, and the + ... GCP oauth exchange and CRM policy read. No ORDER BY, + ... because a streamed relation never reaches the SQL backend. + [Setup] Write Gcp Service Account ${OMNISDK_MOCK_GCP_SA_HOST} + [Teardown] Remove Preview Mock Environment + ${expected} = OperatingSystem.Get File + ... ${CURDIR}${/}..${/}..${/}assets${/}expected${/}preview${/}omni-iam-principals.jsonl + ${gcp_sa} = Set Variable If "${EXECUTION_PLATFORM}" == "docker" + ... /opt/test/tmp/omnisdk-gcp-sa.json ${OMNISDK_MOCK_GCP_SA_HOST} + Set Environment Variable AWS_ACCESS_KEY_ID AK + Set Environment Variable AWS_SECRET_ACCESS_KEY SK + Set Environment Variable AZURE_TENANT_ID mock-tenant + Set Environment Variable AZURE_CLIENT_ID mock-client + Set Environment Variable AZURE_CLIENT_SECRET mock-secret + Set Environment Variable GOOGLE_APPLICATION_CREDENTIALS ${gcp_sa} + ${mock} = Set Variable {"scheme":"http","host":"${LOCAL_HOST_ALIAS}","port":"${MOCKSERVER_PORT_OMNISDK}"} + ${preview} = Catenate SEPARATOR= + ... {"endpoint":{"aws.iam":${mock}, + ... "azure.login":${mock},"azure.graph":${mock}, + ... "gcp.oauth":${mock},"gcp.crm":${mock}}} + ${query} = Catenate SEPARATOR=${SPACE} + ... select * from stackql_preview.audit.omni_iam_principals + ... where region = 'us-east-1' and google_org = '123456789' and method = 'access'; + Should StackQL Exec Inline Jsonl Set Equal + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_MOCKED_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${query} + ... ${expected} + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}Preview-Omni-Iam-Principals-Jsonl-Row-Set-Matches-Expectation.tmp + ... stderr=${CURDIR}${/}tmp${/}Preview-Omni-Iam-Principals-Jsonl-Row-Set-Matches-Expectation-stderr.tmp + + + +Unstable Github Org Members Jsonl Row Set Matches Expectation + [Documentation] A document-derived SELECT: the provider document the + ... registry ships, run against the existing github mock. The + ... document declares no ordering, so the rows are asserted as + ... an unordered set, declared here in an order the query does + ... not emit. + [Teardown] Remove Preview Mock Environment + ${preview} = Catenate SEPARATOR= + ... {"endpoint":"https://${LOCAL_HOST_ALIAS}:${MOCKSERVER_PORT_GITHUB}", + ... "insecureSkipTLSVerify":true,"unstable":true} + ${expected} = Catenate SEPARATOR=\n + ... {"id":"1","login":"some-jimbo-10","type":"User"} + ... {"id":"1","login":"some-jimbo-9","type":"User"} + ... {"id":"1","login":"some-jimbo-8","type":"User"} + ... {"id":"1","login":"some-jimbo-7","type":"User"} + ... {"id":"1","login":"some-jimbo-6","type":"User"} + ... {"id":"1","login":"some-jimbo-5","type":"User"} + ... {"id":"1","login":"some-jimbo-4","type":"User"} + ... {"id":"1","login":"some-jimbo-3","type":"User"} + ... {"id":"1","login":"some-jimbo-2","type":"User"} + ... {"id":"1","login":"some-jimbo-1","type":"User"} + ${query} = Catenate SEPARATOR=${SPACE} + ... select login, id, type from stackql_unstable_github.orgs.members + ... where org = 'dummyorg'; + Should StackQL Exec Inline Jsonl Set Equal + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${query} + ... ${expected} + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members.tmp + ... stderr=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members-stderr.tmp + +Unstable Google Kms Key Rings Jsonl Row Set Matches Expectation + [Documentation] The document-derived counterpart of the registry-backed + ... `google.cloudkms.key_rings` listing, run against the same + ... google mock. The document declares no ordering, so rows + ... are asserted as an unordered set. + [Setup] Write Gcp Service Account ${OMNISDK_MOCK_GCP_SA_HOST} + [Teardown] Remove Preview Mock Environment + ${gcp_sa} = Set Variable If "${EXECUTION_PLATFORM}" == "docker" + ... /opt/test/tmp/omnisdk-gcp-sa.json ${OMNISDK_MOCK_GCP_SA_HOST} + Set Environment Variable GOOGLE_APPLICATION_CREDENTIALS ${gcp_sa} + ${preview} = Catenate SEPARATOR= + ... {"endpoint":"https://${LOCAL_HOST_ALIAS}:${MOCKSERVER_PORT_GOOGLE}", + ... "insecureSkipTLSVerify":true,"unstable":true} + # The suite's google credentials are PKCS1; omnisdk needs PKCS8, so this + # points at the key the setup generates. + ${auth} = Set Variable {"google":{"credentialsfilepath":"${gcp_sa}"}} + ${query} = Catenate SEPARATOR=${SPACE} + ... select name, createTime from stackql_unstable_google.cloudkms.key_rings + ... where projectsId = 'testing-project' and locationsId = 'global'; + ${expected} = Catenate SEPARATOR= + ... {"name":"projects/testing-project/locations/global/keyRings/testing", + ... "createTime":"2022-02-02T02:02:02.02000000Z"} + Should StackQL Exec Inline Jsonl Set Equal + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${auth} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${query} + ... ${expected} + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}Unstable-Google-Kms-Key-Rings.tmp + ... stderr=${CURDIR}${/}tmp${/}Unstable-Google-Kms-Key-Rings-stderr.tmp