diff --git a/AGENTS.md b/AGENTS.md index ebd506a..60acdc1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,9 +67,13 @@ before changing a command. ## Refreshing the bundled ODPS schema -The ODPS JSON Schema is vendored at -`dataproduct/schemas/odps-1.0.0.schema.json` from -`https://raw.githubusercontent.com/bitol-io/open-data-product-standard/main/schema/odps-json-schema-v1.0.0.json`. +The ODPS JSON Schemas are vendored at `dataproduct/schemas/odps-.schema.json`; +`dataproduct/schemas/download` refreshes them (parallel to datacontract-cli's +`datacontract/schemas/download`). `lint` picks the bundled schema by the +document's `apiVersion` (`ODPS_SCHEMA_VERSIONS` in `dataproduct/lint/schema.py`). +A new ODPS release means: add it to `download` and run it, register it in +`ODPS_SCHEMA_VERSIONS`, bump `DEFAULT_ODPS_SCHEMA_VERSION`, and move the init +template to the new version. ## Release diff --git a/CHANGELOG.md b/CHANGELOG.md index e07c56b..c88ab9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ what changed (user-facing). ## [Unreleased] +- Support ODPS v1.1.0: `lint` validates against the bundled v1.1.0 JSON Schema (`type`, `context`, `synonyms`, `deprecated`, `customProperties[].vendor`, element `id`s, optional port `version`/`contractId`) +- `lint` validates against the bundled JSON Schema for the `apiVersion` the document declares (`v1.1.0` → v1.1.0 schema; `v1.0.0`/`v0.9.0` → v1.0.0 schema; unknown → newest), and check names state which schema ran +- `init` template now uses `apiVersion: v1.1.0` + ## [0.1.0] - `init` command: create a valid `dataproduct.odps.yaml` from a bundled ODPS v1.0.0 template diff --git a/README.md b/README.md index cd62023..7e39b44 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ The `dataproduct` CLI is an open-source command-line tool for working with **data products** defined with the -[Open Data Product Standard (ODPS)](https://bitol-io.github.io/open-data-product-standard/v1.0.0/). +[Open Data Product Standard (ODPS)](https://bitol-io.github.io/open-data-product-standard/v1.1.0/). It is the data-product sibling of [`datacontract-cli`](https://github.com/datacontract/datacontract-cli) (which @@ -41,8 +41,10 @@ dataproduct lint --output-format junit --output TEST-dataproduct.xml dataproduct lint --json-schema ./odps.schema.json # validate against a custom schema ``` -Validation is schema-only in 0.1: the data product is checked against the -bundled ODPS v1.0.0 JSON Schema. Exit code is `0` when valid, `1` otherwise. +Validation is schema-only: the data product is checked against the bundled +ODPS JSON Schema matching its `apiVersion` (`v1.1.0`, `v1.0.0`, or `v0.9.0`; +unknown versions are validated against the latest). Exit code is `0` when +valid, `1` otherwise. ### `publish` — publish to Entropy Data diff --git a/dataproduct/data_product.py b/dataproduct/data_product.py index 1d5b2c5..efd0ae1 100644 --- a/dataproduct/data_product.py +++ b/dataproduct/data_product.py @@ -11,7 +11,7 @@ from dataproduct.config import Config from dataproduct.integration.entropy_data import publish_data_product_to_entropy_data from dataproduct.lint.files import read_resource -from dataproduct.lint.schema import fetch_schema +from dataproduct.lint.schema import fetch_schema, schema_version_for from dataproduct.lint.validate import parse_yaml, validate_against_schema from dataproduct.model.exceptions import DataProductException from dataproduct.model.run import Check, ResultEnum, Run @@ -46,15 +46,16 @@ def _load_dict(self) -> dict: return parse_yaml(content) def lint(self) -> Run: - """Validate the data product against the ODPS JSON Schema (schema-only).""" + """Validate the data product against the ODPS JSON Schema matching its ``apiVersion`` (schema-only).""" run = Run.create_run() run.log_info("Linting data product") try: data = self._load_dict() run.dataProductId = data.get("id") run.dataProductVersion = data.get("version") - schema = fetch_schema(self._schema_location) - checks = validate_against_schema(data, schema, self._all_errors) + schema_version = None if self._schema_location else schema_version_for(data.get("apiVersion")) + schema = fetch_schema(self._schema_location, schema_version) + checks = validate_against_schema(data, schema, self._all_errors, schema_version) if checks: run.checks.extend(checks) for check in checks: @@ -64,7 +65,9 @@ def lint(self) -> Run: Check( type="lint", result=ResultEnum.passed, - name="Data product is syntactically valid", + name="Data product is syntactically valid" + if schema_version is None + else f"Data product is valid against ODPS v{schema_version}", ) ) except DataProductException as e: diff --git a/dataproduct/init/init_template.py b/dataproduct/init/init_template.py index e235682..4789aaa 100644 --- a/dataproduct/init/init_template.py +++ b/dataproduct/init/init_template.py @@ -3,7 +3,7 @@ import requests -DEFAULT_DATA_PRODUCT_INIT_TEMPLATE = "odps-1.0.0.init.yaml" +DEFAULT_DATA_PRODUCT_INIT_TEMPLATE = "odps-1.1.0.init.yaml" def get_init_template(location: str = None) -> str: diff --git a/dataproduct/lint/schema.py b/dataproduct/lint/schema.py index 10ac03a..3fff9ae 100644 --- a/dataproduct/lint/schema.py +++ b/dataproduct/lint/schema.py @@ -3,26 +3,46 @@ import logging import os from pathlib import Path -from typing import Any, Dict, Union +from typing import Any, Dict, Optional, Union import requests from dataproduct.model.exceptions import DataProductException from dataproduct.model.run import ResultEnum -DEFAULT_DATA_PRODUCT_SCHEMA = "odps-1.0.0.schema.json" +# ODPS v1.1.0 relaxed several required fields (e.g. `status`, port `version`/`contractId`) +# and added new ones, so older documents must be validated against their own schema. +# v0.9.0 has no dedicated bundled schema; the v1.0.0 schema accepts it. +ODPS_SCHEMA_VERSIONS = { + "v1.1.0": "1.1.0", + "v1.0.0": "1.0.0", + "v0.9.0": "1.0.0", +} +DEFAULT_ODPS_SCHEMA_VERSION = "1.1.0" -def fetch_schema(location: Union[str, Path] = None) -> Dict[str, Any]: +def schema_version_for(api_version: Any = None) -> str: + """Return the bundled ODPS schema version for a document's ``apiVersion``. + + Unknown or missing versions fall back to the newest bundled schema, which + then reports the invalid ``apiVersion`` as a schema violation. + """ + if isinstance(api_version, str): + return ODPS_SCHEMA_VERSIONS.get(api_version, DEFAULT_ODPS_SCHEMA_VERSION) + return DEFAULT_ODPS_SCHEMA_VERSION + + +def fetch_schema(location: Union[str, Path] = None, schema_version: Optional[str] = None) -> Dict[str, Any]: """Fetch the ODPS JSON Schema to validate against. - ``None`` uses the bundled ODPS v1.0.0 schema; otherwise ``location`` is a URL - or local path. + ``None`` uses the bundled schema for ``schema_version`` (newest when + omitted); otherwise ``location`` is a URL or local path. """ if location is None: - logging.info("Use default bundled schema " + DEFAULT_DATA_PRODUCT_SCHEMA) + schema_name = f"odps-{schema_version or DEFAULT_ODPS_SCHEMA_VERSION}.schema.json" + logging.info("Use default bundled schema " + schema_name) schemas = resources.files("dataproduct") - schema_file = schemas.joinpath("schemas", DEFAULT_DATA_PRODUCT_SCHEMA) + schema_file = schemas.joinpath("schemas", schema_name) with schema_file.open("r") as file: return json.load(file) diff --git a/dataproduct/lint/validate.py b/dataproduct/lint/validate.py index d835dd2..5ec556e 100644 --- a/dataproduct/lint/validate.py +++ b/dataproduct/lint/validate.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import yaml from jsonschema.validators import validator_for @@ -32,11 +32,15 @@ def parse_yaml(content: str) -> Dict[str, Any]: return data -def validate_against_schema(data: Dict[str, Any], schema: Dict[str, Any], all_errors: bool = False) -> List[Check]: +def validate_against_schema( + data: Dict[str, Any], schema: Dict[str, Any], all_errors: bool = False, schema_version: Optional[str] = None +) -> List[Check]: """Validate ``data`` against the ODPS JSON Schema. Returns a list of ``error`` checks — empty when the document is valid. With - ``all_errors=False`` (default) only the first violation is reported. + ``all_errors=False`` (default) only the first violation is reported. Check + names state the bundled ``schema_version`` that ran; ``None`` means a custom + schema was supplied and no version is named. """ validator_cls = validator_for(schema) validator_cls.check_schema(schema) @@ -46,16 +50,21 @@ def validate_against_schema(data: Dict[str, Any], schema: Dict[str, Any], all_er if not all_errors: errors = errors[:1] + name = ( + "Check that data product YAML is valid" + if schema_version is None + else f"Check that data product is valid against ODPS v{schema_version}" + ) checks: List[Check] = [] for error in errors: - path = "/".join(str(p) for p in error.absolute_path) or "(root)" + path = "/".join(str(p) for p in error.absolute_path) checks.append( Check( type="lint", result=ResultEnum.error, - name=f"Schema validation failed at '{path}'", - reason=error.message, - field=path, + name=name, + reason=f"{path}: {error.message}" if path else error.message, + field=path or "(root)", ) ) return checks diff --git a/dataproduct/schemas/download b/dataproduct/schemas/download new file mode 100755 index 0000000..976032d --- /dev/null +++ b/dataproduct/schemas/download @@ -0,0 +1,9 @@ +#!/bin/bash +# Refresh the vendored ODPS JSON Schemas (newest from its release tag, older ones from main, +# as datacontract-cli does). After adding a version here, register it in ODPS_SCHEMA_VERSIONS +# (dataproduct/lint/schema.py). +set -e +cd "$(dirname "$0")" + +curl -o odps-1.0.0.schema.json https://raw.githubusercontent.com/bitol-io/open-data-product-standard/refs/heads/main/schema/odps-json-schema-v1.0.0.json +curl -o odps-1.1.0.schema.json https://raw.githubusercontent.com/bitol-io/open-data-product-standard/refs/tags/v1.1.0/schema/odps-json-schema-v1.1.0.json diff --git a/dataproduct/schemas/odps-1.0.0.init.yaml b/dataproduct/schemas/odps-1.1.0.init.yaml similarity index 89% rename from dataproduct/schemas/odps-1.0.0.init.yaml rename to dataproduct/schemas/odps-1.1.0.init.yaml index 6081e2b..ad7a852 100644 --- a/dataproduct/schemas/odps-1.0.0.init.yaml +++ b/dataproduct/schemas/odps-1.1.0.init.yaml @@ -1,9 +1,10 @@ -apiVersion: v1.0.0 +apiVersion: v1.1.0 kind: DataProduct id: my-data-product-id name: My Data Product version: v1.0.0 status: draft +# type: consumerAligned # sourceAligned | aggregate | consumerAligned description: purpose: Purpose of the data product. diff --git a/dataproduct/schemas/odps-1.1.0.schema.json b/dataproduct/schemas/odps-1.1.0.schema.json new file mode 100644 index 0000000..3384d24 --- /dev/null +++ b/dataproduct/schemas/odps-1.1.0.schema.json @@ -0,0 +1,717 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "title": "Open Data Product Standard (ODPS)", + "description": "An open data product standard descriptor to enable defining data products.", + "type": "object", + "required": ["apiVersion", "kind", "id"], + "additionalProperties": false, + "properties": { + "apiVersion": { + "type": "string", + "default": "v1.1.0", + "description": "Version of the standard used to build data product. Default value is v1.1.0.", + "enum": ["v1.1.0", "v1.0.0", "v0.9.0"] + }, + "kind": { + "type": "string", + "default": "DataProduct", + "description": "The kind of file this is. Valid value is `DataProduct`.", + "enum": ["DataProduct"] + }, + "id": { + "type": "string", + "description": "A unique identifier used to reduce the risk of dataset name collisions, such as a UUID." + }, + "name": { + "type": "string", + "description": "Name of the data product." + }, + "deprecated": { + "type": "boolean", + "description": "Indicates this data product is deprecated and should not be used in new implementations. Defaults to false.", + "default": false + }, + "synonyms": { + "$ref": "#/$defs/Synonyms" + }, + "version": { + "type": "string", + "description": "Current version of the data product. Not required, but highly recommended." + }, + "status": { + "type": "string", + "description": "Current status of the data product.", + "examples": ["proposed", "draft", "active", "deprecated", "retired"] + }, + "domain": { + "type": "string", + "description": "Business domain" + }, + "type": { + "type": "string", + "description": "Architectural type of the data product. Common values: `sourceAligned`, `aggregate`, `consumerAligned`. Organizations may define custom types." + }, + "context": { + "$ref": "#/$defs/Context" + }, + "tenant": { + "type": "string", + "description": "Organization identifier" + }, + "authoritativeDefinitions": { + "type": "array", + "description": "List of links to sources that provide more details on the data contract.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + }, + "description": { + "$ref": "#/$defs/Description" + }, + "customProperties": { + "type": "array", + "description": "A list of key/value pairs for custom properties.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "inputPorts": { + "type": "array", + "description": "List of objects describing an input port. You need at least one as a data product needs to get data somewhere.", + "items": { + "$ref": "#/$defs/InputPort" + } + }, + "outputPorts": { + "type": "array", + "description": "List of objects describing an output port. You need at least one, as a data product without output is useless.", + "items": { + "$ref": "#/$defs/OutputPort" + } + }, + "managementPorts": { + "type": "array", + "description": "Management ports define access points for managing the data product.", + "items": { + "$ref": "#/$defs/ManagementPort" + } + }, + "support": { + "type": "array", + "description": "Support and communication channels.", + "items": { + "$ref": "#/$defs/Support" + } + }, + "team": { + "$ref": "#/$defs/Team" + }, + "productCreatedTs": { + "type": "string", + "format": "date-time", + "description": "Timestamp in UTC of when the data product was created, using ISO 8601." + } + }, + "$defs": { + "Tags": { + "type": "array", + "description": "A list of tags that may be assigned to the elements (object or property); the tags keyword may appear at any level. Tags may be used to better categorize an element. For example, `finance`, `sensitive`, `employee_record`.", + "examples": ["finance", "sensitive", "employee_record"], + "items": { + "type": "string" + } + }, + "Description": { + "type": "object", + "description": "Object containing the descriptions.", + "additionalProperties": false, + "properties": { + "purpose": { + "type": "string", + "description": "Intended purpose for the provided data." + }, + "limitations": { + "type": "string", + "description": "Technical, compliance, and legal limitations for data use." + }, + "usage": { + "type": "string", + "description": "Recommended usage of the data." + }, + "authoritativeDefinitions": { + "type": "array", + "description": "List of links to sources that provide more details on the data contract.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + }, + "customProperties": { + "type": "array", + "description": "A list of key/value pairs for custom properties.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + } + } + }, + "CustomProperty": { + "type": "object", + "description": "A key/value pair for custom properties.", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "A unique identifier for the element used to create stable, refactor-safe references. Recommended for elements that will be referenced." + }, + "property": { + "type": "string", + "description": "The name of the key. Names should be in camel case, the same as if they were permanent properties in the contract." + }, + "value": { + "description": "The value of the key." + }, + "description": { + "type": "string", + "description": "Optional description." + }, + "vendor": { + "type": "string", + "description": "Identifies the vendor, provider, or external system associated with this custom property. SHOULD be a stable, lowercase identifier matching ^[a-z0-9][a-z0-9-]*$ (e.g. confluent, zeenea, atlan, soda). Tools MUST preserve unknown vendor values.", + "examples": ["confluent", "zeenea", "atlan", "soda"] + } + }, + "required": ["property", "value"] + }, + "Synonyms": { + "type": "array", + "description": "A list of alternative names for the object. See RFC 0041.", + "items": { + "$ref": "#/$defs/Synonym" + } + }, + "Synonym": { + "type": "object", + "description": "An alternative name for a named object, helping catalogs, AI/LLM tools, and natural language interfaces resolve business vocabulary to the underlying object. See RFC 0041.", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "A unique identifier for the synonym, useful when referencing or deduplicating synonyms across tools. Recommended for elements that will be referenced." + }, + "synonym": { + "type": "string", + "description": "The synonymous term." + }, + "description": { + "type": "string", + "description": "Short human-readable note about when or why this synonym is used." + }, + "locale": { + "type": "string", + "description": "BCP 47 language tag (e.g., `en-US`, `fr-FR`) when the synonym is language-specific." + }, + "source": { + "type": "string", + "description": "Origin of the synonym (e.g., `glossary`, `finance-team`, `legacy-system`)." + }, + "status": { + "type": "string", + "description": "Lifecycle status of the synonym (e.g., `active`, `deprecated`)." + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + } + }, + "required": ["synonym"] + }, + "AuthoritativeDefinition": { + "type": "object", + "description": "A type/link pair for authoritative definitions.", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "A unique identifier for the element used to create stable, refactor-safe references. Recommended for elements that will be referenced." + }, + "type": { + "type": "string", + "description": "Type of definition for authority.", + "examples": ["businessDefinition", "canonicalUrl", "glossary", "implementation", "ontology", "taxonomy", "transformationImplementation", "tutorial", "videoTutorial"] + }, + "url": { + "type": "string", + "format": "uri", + "description": "URL to the authority." + }, + "description": { + "type": "string", + "description": "Optional description." + } + }, + "required": ["type", "url"] + }, + "InputPort": { + "type": "object", + "description": "An input port describing expectations.", + "additionalProperties": false, + "properties": { + "deprecated": { + "type": "boolean", + "description": "Indicates this input port is deprecated and should not be used in new implementations. Defaults to false.", + "default": false + }, + "id": { + "type": "string", + "description": "A unique identifier for the element used to create stable, refactor-safe references. Recommended for elements that will be referenced." + }, + "name": { + "type": "string", + "description": "Name of the input port." + }, + "version": { + "type": "string", + "description": "Version of the input port." + }, + "contractId": { + "type": "string", + "description": "Contract ID for the input port." + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + } + }, + "required": ["name"] + }, + "OutputPort": { + "type": "object", + "description": "An output port describing promises.", + "additionalProperties": false, + "properties": { + "deprecated": { + "type": "boolean", + "description": "Indicates this output port is deprecated and should not be used in new implementations. Defaults to false.", + "default": false + }, + "id": { + "type": "string", + "description": "A unique identifier for the element used to create stable, refactor-safe references. Recommended for elements that will be referenced." + }, + "name": { + "type": "string", + "description": "Name of the output port." + }, + "description": { + "type": "string", + "description": "Human readable short description of the output port." + }, + "type": { + "type": "string", + "description": "There can be different types of output ports, each automated and handled differently. Here you can indicate the type." + }, + "version": { + "type": "string", + "description": "For each version, a different instance of the output port is listed. The combination of the name and version is the key." + }, + "contractId": { + "type": "string", + "description": "Contract ID for the output port." + }, + "sbom": { + "type": "array", + "description": "The SBOM can/should be at the version level.", + "items": { + "$ref": "#/$defs/SBOM" + } + }, + "inputContracts": { + "type": "array", + "description": "Dependencies or input contracts.", + "items": { + "$ref": "#/$defs/InputContract" + } + }, + "synonyms": { + "$ref": "#/$defs/Synonyms" + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + }, + "context": { + "$ref": "#/$defs/Context" + } + }, + "required": ["name"] + }, + "SBOM": { + "type": "object", + "description": "Software Bill of Materials.", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "A unique identifier for the element used to create stable, refactor-safe references. Recommended for elements that will be referenced." + }, + "type": { + "type": "string", + "default": "external", + "description": "Type of SBOM." + }, + "url": { + "type": "string", + "format": "uri", + "description": "URL to the SBOM." + }, + "tags": { "$ref": "#/$defs/Tags" }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { "$ref": "#/$defs/CustomProperty" } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { "$ref": "#/$defs/AuthoritativeDefinition" } + } + } + }, + "InputContract": { + "type": "object", + "description": "Input contract dependency.", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "Contract ID or contractId." + }, + "version": { + "type": "string", + "description": "Version of the input contract." + } + }, + "required": ["id", "version"] + }, + "ManagementPort": { + "type": "object", + "description": "Management port for managing the data product.", + "additionalProperties": false, + "properties": { + "deprecated": { + "type": "boolean", + "description": "Indicates this management port is deprecated and should not be used in new implementations. Defaults to false.", + "default": false + }, + "id": { + "type": "string", + "description": "A unique identifier for the element used to create stable, refactor-safe references. Recommended for elements that will be referenced." + }, + "name": { + "type": "string", + "description": "Endpoint identifier or unique name." + }, + "content": { + "type": "string", + "description": "Content type.", + "examples": ["discoverability", "observability", "control", "dictionary"] + }, + "type": { + "type": "string", + "default": "rest", + "description": "Type: can be `rest` or `topic`. Default is `rest`.", + "examples": ["rest", "topic"] + }, + "url": { + "type": "string", + "format": "uri", + "description": "URL to access the endpoint." + }, + "channel": { + "type": "string", + "description": "Channel to communicate with the data product." + }, + "description": { + "type": "string", + "description": "Purpose and usage." + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + } + }, + "required": ["name", "content"] + }, + "Support": { + "type": "object", + "description": "Support channel.", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "A unique identifier for the element used to create stable, refactor-safe references. Recommended for elements that will be referenced." + }, + "channel": { + "type": "string", + "description": "Channel name or identifier." + }, + "url": { + "type": "string", + "format": "uri", + "description": "Access URL using normal URL scheme (https, mailto, etc.)." + }, + "description": { + "type": "string", + "description": "Description of the channel, free text." + }, + "tool": { + "type": "string", + "description": "Name of the tool.", + "examples": ["email", "slack", "teams", "discord", "ticket", "other"] + }, + "scope": { + "type": "string", + "description": "Scope can be: `interactive`, `announcements`, `issues`.", + "examples": ["interactive", "announcements", "issues"] + }, + "invitationUrl": { + "type": "string", + "format": "uri", + "description": "Some tools uses invitation URL for requesting or subscribing. Follows the URL scheme." + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + } + }, + "required": ["channel", "url"] + }, + "TeamMember": { + "type": "object", + "description": "Team member information.", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "A unique identifier for the element used to create stable, refactor-safe references. Recommended for elements that will be referenced." + }, + "username": { + "type": "string", + "description": "The user's username or email." + }, + "name": { + "type": "string", + "description": "The user's name." + }, + "description": { + "type": "string", + "description": "The user's description." + }, + "role": { + "type": "string", + "description": "The user's job role; Examples might be owner, data steward. There is no limit on the role." + }, + "dateIn": { + "type": "string", + "format": "date", + "description": "The date when the user joined the team." + }, + "dateOut": { + "type": "string", + "format": "date", + "description": "The date when the user ceased to be part of the team." + }, + "replacedByUsername": { + "type": "string", + "description": "The username of the user who replaced the previous user." + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + } + }, + "required": ["username"] + }, + "Context": { + "type": "object", + "description": "AI and semantic context block (RFC-0038). Provides structured guidance for AI agents, LLMs, BI tools, and semantic layer platforms. Optional and additive. Applicable at the data product (top level) and output port levels. Input ports refer to the linked ODCS contract's context instead.", + "additionalProperties": false, + "properties": { + "instructions": { + "type": "string", + "description": "Natural language guidance for AI agents and tools on how to use this entity. Equivalent to a system prompt scoped to this level." + }, + "verifiedStatements": { + "type": "array", + "description": "Canonical business questions, each with an optional curated answer. Entries with `answer` should be returned verbatim by AI agents when a query is semantically close; entries without `answer` serve as sample questions for text-to-SQL priming and disambiguation.", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "Stable identifier for the entry, useful when referencing or deduplicating verified statements across tools." + }, + "question": { + "type": "string", + "description": "The canonical question." + }, + "answer": { + "type": "string", + "description": "The expected response or result description. Optional — omit to signal an unanswered sample question." + }, + "authoritativeDefinitions": { + "type": "array", + "items": { "$ref": "#/$defs/AuthoritativeDefinition" } + }, + "tags": { "$ref": "#/$defs/Tags" }, + "customProperties": { + "type": "array", + "items": { "$ref": "#/$defs/CustomProperty" } + } + }, + "required": ["question"] + } + }, + "constraints": { + "type": "array", + "description": "Negative guidance: what AI agents must NOT do with this entity.", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "Stable identifier for the constraint." + }, + "constraint": { + "type": "string", + "description": "The constraint text (negative guidance for AI agents)." + }, + "authoritativeDefinitions": { + "type": "array", + "items": { "$ref": "#/$defs/AuthoritativeDefinition" } + }, + "tags": { "$ref": "#/$defs/Tags" }, + "customProperties": { + "type": "array", + "items": { "$ref": "#/$defs/CustomProperty" } + } + }, + "required": ["constraint"] + } + } + } + }, + "Team": { + "type": "object", + "description": "Team information.", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Team name." + }, + "description": { + "type": "string", + "description": "Team description." + }, + "members": { + "type": "array", + "description": "List of members.", + "items": { + "$ref": "#/$defs/TeamMember" + } + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + } + } + } + } +} \ No newline at end of file diff --git a/specs/001-init.md b/specs/001-init.md index 4de9a1c..051df0f 100644 --- a/specs/001-init.md +++ b/specs/001-init.md @@ -27,7 +27,7 @@ Example: `dataproduct init dataproduct.odps.yaml` `File already exists, use --overwrite to overwrite` and exit code `1`. 2. Resolve the template contents: - No `--template` → use the **bundled** default template - (`dataproduct/schemas/odps-1.0.0.init.yaml`). + (`dataproduct/schemas/odps-1.1.0.init.yaml`). - `--template` is an `http(s)://` URL → fetch its body. - `--template` is a local path → read it. 3. Write the template string to `LOCATION`. @@ -38,8 +38,8 @@ Example: `dataproduct init dataproduct.odps.yaml` ## Bundled default template -- Lives at `dataproduct/schemas/odps-1.0.0.init.yaml`. -- Uses `apiVersion: v1.0.0`, `kind: DataProduct`. +- Lives at `dataproduct/schemas/odps-1.1.0.init.yaml`. +- Uses `apiVersion: v1.1.0`, `kind: DataProduct`, with a commented `type` stub. - Contains a **static** `id` placeholder (`my-data-product-id`) — see decision below. - Includes at least one `outputPort` so the result satisfies the best-practice @@ -48,7 +48,7 @@ Example: `dataproduct init dataproduct.odps.yaml` Proposed content: ```yaml -apiVersion: v1.0.0 +apiVersion: v1.1.0 kind: DataProduct id: my-data-product-id name: My Data Product diff --git a/specs/002-lint.md b/specs/002-lint.md index ecb058f..37efe79 100644 --- a/specs/002-lint.md +++ b/specs/002-lint.md @@ -34,14 +34,20 @@ Run as an ordered set of checks, each producing a result entry: 1. **File is readable** — LOCATION exists / URL fetches; else `error`. 2. **Valid YAML** — parses to a mapping; else `error`. 3. **Schema validation (authoritative)** — validate the parsed document against - the bundled ODPS JSON Schema (`dataproduct/schemas/odps-1.0.0.schema.json`), - or the `--json-schema` override. Each violation is an `error` (path + + the bundled ODPS JSON Schema matching the document's `apiVersion` + (`v1.1.0` → `odps-1.1.0.schema.json`; `v1.0.0` and `v0.9.0` → + `odps-1.0.0.schema.json`; unknown/missing → latest, which then reports the + bad `apiVersion`), or the `--json-schema` override. Each violation is an `error` (path + message). With `--all-errors`, collect every violation; otherwise stop at the first. - - This enforces the strictly-required fields (`apiVersion`, `kind`, `id`, - `status`), the `kind: DataProduct` / `apiVersion` enums (both `v0.9.0` and - `v1.0.0` are accepted, silently — no version special-casing), and the - required subfields of ports/support/etc. + - This enforces the strictly-required fields (`apiVersion`, `kind`, `id`; + plus `status` for v1.0.0 documents), the `kind: DataProduct` / + `apiVersion` enums (`v0.9.0`, `v1.0.0`, `v1.1.0` are all accepted, silently), + and the required subfields of ports/support/etc. + - Picking the schema by `apiVersion` is deliberate: v1.1.0 both relaxed + required fields (`status`, port `version`/`contractId`) and added new + ones (`type`, `context`, `synonyms`, `deprecated`, `vendor`), so a v1.0.0 + document must not silently pass with v1.1.0 fields or without `status`. **0.1 is schema-only** — parity with datacontract-cli's `lint`, which validates against the JSON Schema and nothing more. Best-practice warnings (≥1 outputPort, @@ -71,12 +77,17 @@ run = DataProduct(data_product_file="dataproduct.odps.yaml").lint() assert run.result == "passed" ``` -## Bundled schema +## Bundled schemas + +- `dataproduct/schemas/odps-1.1.0.schema.json` (default) and + `dataproduct/schemas/odps-1.0.0.schema.json`, vendored from + `https://raw.githubusercontent.com/bitol-io/open-data-product-standard/main/schema/odps-json-schema-v.json`. +- Selection lives in `ODPS_SCHEMA_VERSIONS` (`dataproduct/lint/schema.py`); + `dataproduct/schemas/download` refreshes the vendored copies. +- Check names state the schema that ran (`Data product is valid against ODPS + v1.1.0` / `Check that data product is valid against ODPS v1.0.0`); with a + custom `--json-schema` no version is named. -- `dataproduct/schemas/odps-1.0.0.schema.json`, vendored from - `https://raw.githubusercontent.com/bitol-io/open-data-product-standard/main/schema/odps-json-schema-v1.0.0.json`. -- A small maintenance script (`update_schema.py`, parallel to datacontract-cli's - update scripts) can refresh the vendored copy. ## Acceptance criteria @@ -92,6 +103,10 @@ assert run.result == "passed" - [ ] `--json-schema ` validates against the supplied schema instead of the bundled one. - [ ] A missing file yields a clean `error` result (no traceback), exit `1`. +- [ ] A `v1.1.0` document using `type`, `context`, `synonyms`, `deprecated`, + `vendor`, and ports without `version`/`contractId` returns `passed`. +- [ ] A `v1.1.0` document without `status` returns `passed`; a `v1.0.0` one fails. +- [ ] A `v1.0.0` document using a v1.1.0-only field (e.g. `type`) fails. ## Test cases (pytest) @@ -104,6 +119,9 @@ assert run.result == "passed" 7. `test_lint_junit_output` → well-formed XML written to file. 8. `test_lint_custom_json_schema`. 9. `test_lint_missing_file` → error, exit 1. +10. `test_lint_valid_v1_1_0`, `test_lint_v1_1_0_status_is_optional`, + `test_lint_v1_1_0_fields_rejected_under_v1_0_0`, + `test_lint_names_the_schema_that_ran`, `test_schema_version_selected_by_api_version`. ## Decisions @@ -111,6 +129,12 @@ assert run.result == "passed" datacontract-cli. Best-practice warnings backlogged ([backlog.md](backlog.md)). 2. **Reference resolution:** ✅ **Deferred** — no inlining of `authoritativeDefinitions` in 0.1 ([backlog.md](backlog.md)). -3. **`apiVersion v0.9.0`:** ✅ **Accept both silently** (schema default; no - special-casing). A deprecation warning is backlogged. +3. **`apiVersion v0.9.0`:** ✅ **Accept silently** (validated with the v1.0.0 + schema, which has no dedicated v0.9.0 rules). A deprecation warning is backlogged. +4. **ODPS v1.1.0 (2026-09):** ✅ **Schema chosen per `apiVersion`**, mirroring + datacontract-cli #1606 (`lint` validates against the declared ODCS + `apiVersion`; older versions without their own schema share the nearest + one, unknown versions fall back to the newest). Needed here because v1.1.0 + loosened required fields, so a v1.0.0 document must not pass without + `status` or with v1.1.0-only fields. ``` \ No newline at end of file diff --git a/specs/backlog.md b/specs/backlog.md index e72d227..9158fe6 100644 --- a/specs/backlog.md +++ b/specs/backlog.md @@ -30,8 +30,8 @@ Do this in both CLIs (datacontract-cli is also schema-only today). - Source: [002-lint.md](002-lint.md) ### 3. `apiVersion v0.9.0` deprecation warning (cross-CLI) -0.1 accepts `v0.9.0` and `v1.0.0` silently. Improvement: warn (non-blocking) -when `v0.9.0` is used, nudging toward `v1.0.0`. Pairs with item 2. +`v0.9.0`, `v1.0.0`, and `v1.1.0` are accepted silently. Improvement: warn +(non-blocking) when `v0.9.0` is used, nudging toward `v1.1.0`. Pairs with item 2. - Source: [002-lint.md](002-lint.md) ### 4. Pre-publish lint gate + `--skip-lint` (cross-CLI) diff --git a/specs/overview.md b/specs/overview.md index 032aff7..f13c802 100644 --- a/specs/overview.md +++ b/specs/overview.md @@ -3,7 +3,7 @@ ## Purpose `dataproduct-cli` is an open-source command-line tool for working with **data -products** defined with the [Open Data Product Standard (ODPS) v1.0.0](https://bitol-io.github.io/open-data-product-standard/v1.0.0/). +products** defined with the [Open Data Product Standard (ODPS) v1.1.0](https://bitol-io.github.io/open-data-product-standard/v1.1.0/). It is the data-product counterpart to [`datacontract-cli`](https://github.com/datacontract/datacontract-cli) (which targets the Open Data **Contract** Standard, ODCS), and deliberately mirrors its @@ -27,31 +27,35 @@ Out of scope for now (candidates for later): `export`, `import`, `changelog`, The default filename is `dataproduct.odps.yaml`. The document is an ODPS `DataProduct`. -### Top-level fields (ODPS v1.0.0) +### Top-level fields (ODPS v1.1.0) | Field | Req. | Notes | |---|---|---| -| `apiVersion` | ✅ | `v1.0.0` (schema also allows `v0.9.0`) | +| `apiVersion` | ✅ | `v1.1.0` (schema also allows `v1.0.0`, `v0.9.0`) | | `kind` | ✅ | must be `DataProduct` | | `id` | ✅ | unique identifier, UUID recommended | -| `status` | ✅ | e.g. `proposed`, `draft`, `active`, `deprecated`, `retired` | +| `status` | — | e.g. `proposed`, `draft`, `active`, `deprecated`, `retired` (required in v1.0.0, optional since v1.1.0) | | `name` | — | human-readable name | | `version` | — | product version (e.g. `v1.0.0`) | +| `type` | — | architectural type, e.g. `sourceAligned`, `aggregate`, `consumerAligned` (v1.1.0) | +| `deprecated` | — | boolean, default `false` (v1.1.0; also on ports) | | `domain` | — | business domain | | `tenant` | — | organization identifier | | `description` | — | object: `purpose`, `usage`, `limitations`, … | | `tags` | — | list of strings | -| `inputPorts` | — | items require `name`, `version`, `contractId` | -| `outputPorts` | — | items require `name`, `version`; best practice ≥ 1 | +| `synonyms` | — | list of `{synonym, locale?, source?, …}` (v1.1.0; also on output ports) | +| `context` | — | AI/semantic context: `instructions`, `verifiedStatements`, `constraints` (v1.1.0; also on output ports) | +| `inputPorts` | — | items require `name` (`version`, `contractId` also required in v1.0.0) | +| `outputPorts` | — | items require `name` (`version` also required in v1.0.0); best practice ≥ 1 | | `managementPorts` | — | management/observability endpoints | | `support` | — | items require `channel`, `url` | | `team` | — | object with `members` | -| `customProperties` | — | list of `{property, value}` | +| `customProperties` | — | list of `{property, value, vendor?}` | | `authoritativeDefinitions` | — | list of `{type, url}` | | `productCreatedTs` | — | ISO 8601 UTC timestamp | -> **Required-field note.** The official JSON Schema strictly requires only -> `apiVersion`, `kind`, `id`, `status`. The prose standard additionally +> **Required-field note.** The official v1.1.0 JSON Schema strictly requires +> only `apiVersion`, `kind`, `id` (v1.0.0 also requires `status`). The prose standard additionally > recommends at least one `outputPort`. `lint` treats the JSON Schema as > authoritative for pass/fail. In 0.1 that's the whole story (schema-only, > parity with datacontract-cli); best-practice warnings like "≥1 outputPort" @@ -60,7 +64,7 @@ The default filename is `dataproduct.odps.yaml`. The document is an ODPS ### Minimal valid example (from the ODPS repo) ```yaml -apiVersion: v1.0.0 +apiVersion: v1.1.0 kind: DataProduct id: 064c4630-8aad-4dc0-ba95-0f69940e6b18 status: active @@ -125,7 +129,7 @@ dataproduct-cli/ │ │ └── entropy_data.py # publish to Entropy Data │ ├── model/ # Pydantic models + Run/result types + exceptions │ ├── output/ # result writers (console, json, junit) -│ └── schemas/ # bundled odps-*.schema.json + *.init.yaml +│ └── schemas/ # bundled odps-.schema.json (one per supported apiVersion) + *.init.yaml └── tests/ ├── fixtures/ └── test_*.py diff --git a/tests/fixtures/lint/missing-status-v1.1.0.odps.yaml b/tests/fixtures/lint/missing-status-v1.1.0.odps.yaml new file mode 100644 index 0000000..11955f9 --- /dev/null +++ b/tests/fixtures/lint/missing-status-v1.1.0.odps.yaml @@ -0,0 +1,4 @@ +apiVersion: v1.1.0 +kind: DataProduct +id: 064c4630-8aad-4dc0-ba95-0f69940e6b18 +name: Missing Status diff --git a/tests/fixtures/lint/v1.1.0-fields-in-v1.0.0.odps.yaml b/tests/fixtures/lint/v1.1.0-fields-in-v1.0.0.odps.yaml new file mode 100644 index 0000000..87ee9ff --- /dev/null +++ b/tests/fixtures/lint/v1.1.0-fields-in-v1.0.0.odps.yaml @@ -0,0 +1,6 @@ +apiVersion: v1.0.0 +kind: DataProduct +id: 064c4630-8aad-4dc0-ba95-0f69940e6b18 +status: active +name: Uses v1.1.0 Fields Under v1.0.0 +type: aggregate diff --git a/tests/fixtures/lint/valid-dataproduct-v1.1.0.odps.yaml b/tests/fixtures/lint/valid-dataproduct-v1.1.0.odps.yaml new file mode 100644 index 0000000..b5cc19e --- /dev/null +++ b/tests/fixtures/lint/valid-dataproduct-v1.1.0.odps.yaml @@ -0,0 +1,53 @@ +apiVersion: v1.1.0 +kind: DataProduct +id: 064c4630-8aad-4dc0-ba95-0f69940e6b18 +status: active +name: Orders Data Product +version: v2.0.0 +type: consumerAligned +deprecated: false + +description: + purpose: Curated orders for analytics + limitations: None + usage: Analytics and reporting + +context: + instructions: Use the orders output port for revenue questions. + verifiedStatements: + - id: q-revenue + question: What was total revenue last month? + answer: Sum order_total over the previous calendar month. + constraints: + - constraint: Do not expose customer email addresses. + +synonyms: + - synonym: Sales Orders + locale: en-US + source: glossary + +inputPorts: + - id: raw-orders + name: raw-orders + +outputPorts: + - id: orders + name: orders + description: Curated orders + type: tables + deprecated: false + synonyms: + - synonym: Bestellungen + locale: de-DE + context: + instructions: Prefer this port over legacy-orders. + sbom: + - type: library + tags: ['build'] + - name: legacy-orders + deprecated: true + +customProperties: + - property: costCenter + value: cc-4711 + vendor: acme diff --git a/tests/test_init.py b/tests/test_init.py index ee8ddcf..464b7b4 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -12,6 +12,7 @@ def test_init_default_creates_valid_file(tmp_path, monkeypatch): assert result.exit_code == 0, result.output created = tmp_path / "dataproduct.odps.yaml" assert created.exists() + assert "apiVersion: v1.1.0" in created.read_text() # The generated file must pass lint with zero errors. run = DataProduct(data_product_file=str(created)).lint() diff --git a/tests/test_lint.py b/tests/test_lint.py index b395e6d..470af09 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -4,6 +4,7 @@ from dataproduct.cli import app from dataproduct.data_product import DataProduct +from dataproduct.lint.schema import fetch_schema, schema_version_for runner = CliRunner() @@ -22,6 +23,60 @@ def test_lint_valid(): assert run.dataProductId == "064c4630-8aad-4dc0-ba95-0f69940e6b18" +def test_lint_valid_v1_1_0(): + run = _lint("valid-dataproduct-v1.1.0.odps.yaml", all_errors=True) + assert run.result == "passed", run.checks + assert run.checks[0].name == "Data product is valid against ODPS v1.1.0" + + +def test_lint_names_the_schema_that_ran(): + run = _lint("valid-dataproduct.odps.yaml") + assert run.result == "passed", run.checks + assert run.checks[0].name == "Data product is valid against ODPS v1.0.0" + + +def test_lint_names_no_version_for_custom_schema(): + run = DataProduct( + data_product_file=str(FIXTURES / "valid-dataproduct.odps.yaml"), + schema_location=str(BUNDLED_SCHEMA), + ).lint() + assert run.checks[0].name == "Data product is syntactically valid" + + +def test_lint_v1_1_0_status_is_optional(): + run = _lint("missing-status-v1.1.0.odps.yaml") + assert run.result == "passed", run.checks + + +def test_lint_v1_1_0_fields_rejected_under_v1_0_0(): + run = _lint("v1.1.0-fields-in-v1.0.0.odps.yaml") + assert run.result == "failed" + assert run.checks[0].name == "Check that data product is valid against ODPS v1.0.0" + assert "type" in (run.checks[0].reason or "") + + +def test_lint_unknown_api_version_reported(): + run = _lint("multiple-errors.odps.yaml", all_errors=True) + assert run.result == "failed" + assert all(c.name == "Check that data product is valid against ODPS v1.1.0" for c in run.checks) + assert "apiVersion" in [c.field for c in run.checks] + + +def test_schema_version_selected_by_api_version(): + assert schema_version_for("v1.1.0") == "1.1.0" + assert schema_version_for("v1.0.0") == "1.0.0" + assert schema_version_for("v0.9.0") == "1.0.0" + assert schema_version_for(None) == "1.1.0" + assert schema_version_for("v9.9.9") == "1.1.0" + assert schema_version_for(["not", "a", "string"]) == "1.1.0" + + +def test_fetch_bundled_schema_per_version(): + assert "status" in fetch_schema(schema_version="1.0.0")["required"] + assert "status" not in fetch_schema(schema_version="1.1.0")["required"] + assert fetch_schema()["properties"]["apiVersion"]["default"] == "v1.1.0" + + def test_lint_valid_cli_exit_zero(): result = runner.invoke(app, ["lint", str(FIXTURES / "valid-dataproduct.odps.yaml")]) assert result.exit_code == 0, result.output