access-audit

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/pflag v1.0.9
 	github.com/spf13/viper v1.10.1
-	github.com/stackql-labs/omnisdk v0.1.0-alpha07
+	github.com/stackql-labs/omnisdk v0.1.1-alpha06
 	github.com/stackql/any-sdk v0.5.4-alpha01
 	github.com/stackql/go-suffix-map v0.0.1-alpha01
 	github.com/stackql/psql-wire v0.1.2-beta01

go.sum

@@ -348,8 +348,8 @@ github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk=
 github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU=
 github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4=
 github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U=
-github.com/stackql-labs/omnisdk v0.1.0-alpha07 h1:kGUtupkwAVZWwCkMQZ4vjCD5jIDPmGCSq/OdH4Dmos0=
-github.com/stackql-labs/omnisdk v0.1.0-alpha07/go.mod h1:SHUOryRWXeZip4JTWeA8iwmz4lYdBSp0nbWXDX29JrU=
+github.com/stackql-labs/omnisdk v0.1.1-alpha06 h1:jVetiuZa6J1uWJ4vPc/YQtol4l7Lgoq281UWvYXw6GM=
+github.com/stackql-labs/omnisdk v0.1.1-alpha06/go.mod h1:WzvNj/bVv53yGFsVJpYWCJC1xAEdmQSFJl9eVpkRpCY=
 github.com/stackql/any-sdk v0.5.4-alpha01 h1:AyjD2Hyk7D8v1fHHtLpfeqQRxKuj7rC71dzi1Zh9LoA=
 github.com/stackql/any-sdk v0.5.4-alpha01/go.mod h1:BiE8uiAJMUa8n4U/yMlhvrVhE2M+5Rt8utBXYkwj9To=
 github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8=

internal/stackql/cmd/root.go

@@ -155,7 +155,8 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(&runtimeCtx.StoreTxnCfgRaw, dto.StoreTxnCfgRawKey, "{}", "JSON / YAML string representing Txn store config")
 	rootCmd.PersistentFlags().StringVar(&runtimeCtx.GCCfgRaw, dto.GCCfgRawKey, "{}", "JSON / YAML string representing GC config")
 	rootCmd.PersistentFlags().StringVar(&runtimeCtx.ACIDCfgRaw, dto.ACIDCfgRawKey, "{}", "JSON / YAML string representing ACID config")
-	rootCmd.PersistentFlags().StringVar(&previewCfgRaw, intrinsic.CfgRawKey, "{}", "JSON string configuring the "+intrinsic.ProviderName+" provider backend; keys: batchSize, flushInterval, endpoint")
+	rootCmd.PersistentFlags().StringVar(&previewCfgRaw, intrinsic.CfgRawKey, "{}", "JSON string configuring the "+intrinsic.ProviderName+
+		" provider backend; keys: batchSize, flushInterval, endpoint, unstable")
 	rootCmd.PersistentFlags().StringVar(&runtimeCtx.SessionCtxRaw, dto.SessionCtxKey, "{}", "JSON / YAML string representing session config")
 	rootCmd.PersistentFlags().IntVar(&runtimeCtx.APIRequestTimeout, dto.APIRequestTimeoutKey, 45, "API request timeout in seconds, 0 for no timeout.") //nolint:mnd // TODO: investigate
 	rootCmd.PersistentFlags().StringVar(&dummyString, dto.ColorSchemeKey, "", "DEPRECATED: color schems no longer active")

internal/stackql/handler/handler.go

@@ -323,6 +323,15 @@ func (hc *standardHandlerContext) GetSupportedProviders(extended bool) (map[stri
 		} else {
 			retVal[pn] = getProviderMap(pn, pd)
 		}
+		// Every locally available bundle is also addressable document-first,
+		// once that has been opted into.
+		if intrinsic.IsUnstableEnabled() {
+			unstable := intrinsic.UnstablePrefix + pn
+			retVal[unstable] = map[string]interface{}{
+				"name":    unstable,
+				"version": intrinsic.ProviderVersion,
+			}
+		}
 	}
 	return retVal, nil
 }

internal/stackql/intrinsic/doc.go

@@ -0,0 +1,266 @@
+package intrinsic
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/stackql-labs/omnisdk/pkg/docparse/aot"
+	"github.com/stackql-labs/omnisdk/pkg/omnisdk"
+	"github.com/stackql/any-sdk/pkg/dto"
+	"github.com/stackql/any-sdk/public/formulation"
+	"github.com/stackql/psql-wire/pkg/sqldata"
+	"github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto"
+	"github.com/stackql/stackql/internal/stackql/util"
+
+	"github.com/stackql/stackql-parser/go/vt/sqlparser"
+	"gopkg.in/yaml.v2"
+)
+
+// UnstablePrefix names the document-driven providers. The convention is
+// omnisdk's own, and its addresses carry the prefixed provider name, so this is
+// its constant rather than a copy of the literal.
+const UnstablePrefix = aot.DefaultProviderPrefix
+
+// IsUnstableEnabled reports whether the document-driven providers were opted
+// into. They are documents read straight from disk, with none of the registry's
+// curation behind them, so nothing exposes them until a caller asks.
+func IsUnstableEnabled() bool {
+	return previewCfg.getUnstableEnabled()
+}
+
+// docProvider is the bundle behind an unstable provider name, or false.
+func docProvider(name string) (string, bool) {
+	if !IsUnstableEnabled() {
+		return "", false
+	}
+	trimmed := strings.TrimSpace(name)
+	if !strings.HasPrefix(strings.ToLower(trimmed), UnstablePrefix) {
+		return "", false
+	}
+	bundle := trimmed[len(UnstablePrefix):]
+	if bundle == "" {
+		return "", false
+	}
+	return bundle, true
+}
+
+// bundleAliases maps the provider name stackql presents onto the directory the
+// registry actually wrote. Keeping them apart matters: the name a caller typed
+// is the one echoed back, and it is the only one they can address.
+var bundleAliases = map[string]string{ //nolint:gochecknoglobals // fixed mapping
+	"google": "googleapis.com",
+}
+
+func bundleDir(label string) string {
+	if dir, ok := bundleAliases[strings.ToLower(label)]; ok {
+		return dir
+	}
+	return label
+}
+
+// localDocRoot is where provider documents are read from, resolved exactly as
+// the canonical providers resolve it, so an unstable provider reads whatever
+// documents are already on disk rather than needing assets of its own.
+func localDocRoot(runtimeCtx dto.RuntimeCtx) string {
+	var registryCfg formulation.RegistryConfig
+	if err := yaml.Unmarshal([]byte(runtimeCtx.RegistryRaw), &registryCfg); err == nil {
+		if registryCfg.LocalDocRoot != "" {
+			return registryCfg.LocalDocRoot
+		}
+		if strings.HasPrefix(registryCfg.RegistryURL, "file:") {
+			return filepath.Clean(
+				filepath.Join(strings.TrimPrefix(registryCfg.RegistryURL, "file:"), ".."))
+		}
+	}
+	return runtimeCtx.ApplicationFilesRootPath
+}
+
+// docRoot is the bundle's own directory inside stackql's document root. The
+// versioned directory is used rather than the root itself, because a registry
+// root is addressed as "<provider>.<service>.<resource>" and a provider whose
+// name carries a dot ("googleapis.com") cannot be named that way.
+func docRoot(ctx queryContext, bundle string) (string, error) {
+	root := filepath.Join(localDocRoot(ctx.GetRuntimeContext()), "src", bundleDir(bundle))
+	matches, err := filepath.Glob(filepath.Join(root, "*", "provider.yaml"))
+	if err != nil || len(matches) == 0 {
+		return "", fmt.Errorf("no provider document for '%s%s' under '%s'", UnstablePrefix, bundle, root)
+	}
+	sort.Strings(matches)
+	return filepath.Dir(matches[len(matches)-1]), nil
+}
+
+// docServices lists the services a bundle ships documents for.
+func docServices(ctx queryContext, bundle string) ([]string, error) {
+	dir, err := docRoot(ctx, bundle)
+	if err != nil {
+		return nil, err
+	}
+	services, _, err := omnisdk.DocCatalog(dir, bundle)
+	return services, err
+}
+
+// docResourceTables presents a service's resources as relations.
+func docResourceTables(ctx queryContext, bundle, service string) ([]table, error) {
+	dir, err := docRoot(ctx, bundle)
+	if err != nil {
+		return nil, err
+	}
+	resources, err := omnisdk.DocResources(dir, bundle, service)
+	if err != nil {
+		return nil, err
+	}
+	sort.Strings(resources)
+	out := make([]table, 0, len(resources))
+	for _, resource := range resources {
+		out = append(out, table{service: service, name: resource, isData: true})
+	}
+	return out, nil
+}
+
+// docMethods lists a resource's methods as the document declares them.
+func docMethods(ctx queryContext, bundle, service, resource string) ([]relationMethod, error) {
+	dir, err := docRoot(ctx, bundle)
+	if err != nil {
+		return nil, err
+	}
+	methods, err := omnisdk.DocMethods(dir, bundle, service, resource)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]relationMethod, 0, len(methods))
+	for _, method := range methods {
+		out = append(out, relationMethod{name: method.Name, description: method.OperationID})
+	}
+	return out, nil
+}
+
+// docSelectFunc routes a SELECT over a document-driven relation. The address is
+// the bundle's own "<provider>.<service>.<resource>", and the plan it yields
+// streams exactly as a hand-authored one does.
+func docSelectFunc(
+	ctx queryContext,
+	node *sqlparser.Select,
+	bundle, service, resource string,
+) (func() internaldto.ExecutorOutput, bool) {
+	if unsupported := unsupportedClauses(node); len(unsupported) > 0 {
+		return refuse(fmt.Errorf(
+			"relation '%s%s.%s.%s' streams its rows, so %s cannot be applied; remove %s from the query",
+			UnstablePrefix, bundle, service, resource,
+			strings.Join(unsupported, ", "), pluralClause(len(unsupported)))), true
+	}
+	params, badPredicates := equalityPredicates(node.Where)
+	if len(badPredicates) > 0 {
+		return refuse(fmt.Errorf(
+			"relation '%s%s.%s.%s' streams its rows, so only equality predicates are applied; "+
+				"%s cannot be honoured",
+			UnstablePrefix, bundle, service, resource, strings.Join(badPredicates, ", "))), true
+	}
+	address := fmt.Sprintf("%s%s.%s.%s", UnstablePrefix, bundle, service, resource)
+	return func() internaldto.ExecutorOutput {
+		input := previewCfg
+		dir, dirErr := docRoot(ctx, bundle)
+		if dirErr != nil {
+			return internaldto.NewErroneousExecutorOutput(dirErr)
+		}
+		plan, err := omnisdk.NewFromCatalog(dir, address, omnisdk.Args{
+			Params:                params,
+			Auth:                  omnisdkAuth(providerAuthContext(ctx, bundle)),
+			Endpoint:              input.getEndpoint(),
+			InsecureSkipTLSVerify: input.getInsecureSkipTLSVerify(),
+		})
+		if err != nil {
+			return internaldto.NewErroneousExecutorOutput(err)
+		}
+		rows, openErr := plan.Open(context.Background())
+		if openErr != nil {
+			return internaldto.NewErroneousExecutorOutput(openErr)
+		}
+		// A document declares no egress schema, so the columns are those the
+		// first row carries; projection is applied over them.
+		stream := &rowStream{
+			rows:          rows,
+			batchSize:     input.getBatchSize(),
+			flushInterval: input.getFlushInterval(),
+			table:         sqldata.NewSQLTable(0, resource),
+			typCfg:        ctx.GetTypingConfig(),
+			projection:    node.SelectExprs,
+		}
+		primed, readErr := newPrimedStream(stream)
+		if readErr != nil {
+			return internaldto.NewErroneousExecutorOutput(readErr)
+		}
+		return internaldto.NewExecutorOutput(primed, nil, nil, nil, nil)
+	}, true
+}
+
+func refuse(err error) func() internaldto.ExecutorOutput {
+	return func() internaldto.ExecutorOutput {
+		return internaldto.NewErroneousExecutorOutput(err)
+	}
+}
+
+func showDocServices(ctx queryContext, bundle string, extended bool) internaldto.ExecutorOutput {
+	services, err := docServices(ctx, bundle)
+	if err != nil {
+		return internaldto.NewErroneousExecutorOutput(err)
+	}
+	rows := make(map[string]map[string]interface{}, len(services))
+	for i, service := range services {
+		row := map[string]interface{}{"id": service, "name": service, "title": service}
+		if extended {
+			row["description"] = service
+			row["version"] = ProviderVersion
+			row["preferred"] = nil
+		}
+		rows[fmt.Sprintf("%06d", i)] = row
+	}
+	return prepare(ctx, formulation.GetServicesHeader(extended), rows, util.DefaultRowSort)
+}
+
+func showDocResources(
+	ctx queryContext, bundle, service string, extended bool) internaldto.ExecutorOutput {
+	tables, err := docResourceTables(ctx, bundle, service)
+	if err != nil {
+		return internaldto.NewErroneousExecutorOutput(err)
+	}
+	rows := make(map[string]map[string]interface{}, len(tables))
+	for i, tbl := range tables {
+		row := map[string]interface{}{
+			"id":   fmt.Sprintf("%s%s.%s.%s", UnstablePrefix, bundle, service, tbl.name),
+			"name": tbl.name,
+		}
+		if extended {
+			row["description"] = tbl.description
+		}
+		rows[fmt.Sprintf("%06d", i)] = row
+	}
+	return prepare(ctx, formulation.GetResourcesHeader(extended), rows, util.DefaultRowSort)
+}
+
+func showDocMethods(
+	ctx queryContext, bundle, service, resource string, extended bool) internaldto.ExecutorOutput {
+	methods, err := docMethods(ctx, bundle, service, resource)
+	if err != nil {
+		return internaldto.NewErroneousExecutorOutput(err)
+	}
+	columnOrder := []string{"MethodName", "RequiredParams", "SQLVerb"}
+	if extended {
+		columnOrder = append(columnOrder, "description")
+	}
+	rows := make(map[string]map[string]interface{}, len(methods))
+	for i, method := range methods {
+		row := map[string]interface{}{
+			"MethodName":     method.name,
+			"RequiredParams": strings.Join(method.requiredParams, ", "),
+			"SQLVerb":        strings.ToUpper(selectMethodName),
+		}
+		if extended {
+			row["description"] = method.description
+		}
+		rows[fmt.Sprintf("%06d", i)] = row
+	}
+	return prepare(ctx, columnOrder, rows, util.DefaultRowSort)
+}

internal/stackql/intrinsic/intrinsic.go

@@ -35,6 +35,7 @@ type column struct {
 type table struct {
 	service     string
 	name        string
+	isData      bool
 	description string
 	columns     []column
 }
@@ -48,6 +49,7 @@ type queryContext interface {
 	SetCurrentProvider(string)
 	GetTypingConfig() typing.Config
 	GetAuthContext(providerName string) (*dto.AuthCtx, error)
+	GetRuntimeContext() dto.RuntimeCtx
 }
 
 func GeneratePrimitiveFunc(
@@ -76,7 +78,11 @@ func GenerateStreamFunc(
 }
 
 func IsProvider(name string) bool {
-	return strings.EqualFold(strings.TrimSpace(name), ProviderName)
+	if strings.EqualFold(strings.TrimSpace(name), ProviderName) {
+		return true
+	}
+	_, isDoc := docProvider(name)
+	return isDoc
 }
 
 func resolveProvider(providerName string, currentProvider string) string {
@@ -132,17 +138,34 @@ func showFunc(
 	extended := isExtended(node.Extended)
 	switch strings.ToUpper(strings.TrimSpace(node.Type)) {
 	case "SERVICES":
-		if !IsProvider(resolveProvider(node.OnTable.Name.GetRawVal(), currentProvider)) {
+		provider := resolveProvider(node.OnTable.Name.GetRawVal(), currentProvider)
+		if bundle, isDoc := docProvider(provider); isDoc {
+			return func() internaldto.ExecutorOutput { return showDocServices(ctx, bundle, extended) }, true
+		}
+		if !IsProvider(provider) {
 			return nil, false
 		}
 		return func() internaldto.ExecutorOutput { return showServices(ctx, extended) }, true
 	case "RESOURCES":
 		serviceStr := node.OnTable.Name.GetRawVal()
+		if bundle, isDoc := docProvider(resolveProvider(node.OnTable.Qualifier.GetRawVal(), currentProvider)); isDoc {
+			return func() internaldto.ExecutorOutput {
+				return showDocResources(ctx, bundle, serviceStr, extended)
+			}, true
+		}
 		if !isService(node.OnTable.Qualifier.GetRawVal(), serviceStr, currentProvider) {
 			return nil, false
 		}
 		return func() internaldto.ExecutorOutput { return showResources(ctx, serviceStr, extended) }, true
 	case "METHODS":
+		if bundle, isDoc := docProvider(
+			resolveProvider(node.OnTable.QualifierSecond.GetRawVal(), currentProvider)); isDoc {
+			service := node.OnTable.Qualifier.GetRawVal()
+			resource := node.OnTable.Name.GetRawVal()
+			return func() internaldto.ExecutorOutput {
+				return showDocMethods(ctx, bundle, service, resource, extended)
+			}, true
+		}
 		tbl, ok := lookupTable(
 			node.OnTable.QualifierSecond.GetRawVal(),
 			node.OnTable.Qualifier.GetRawVal(),