From 7cb27295516b3983c5750ba271af38a0cdbbd11f Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:14:09 +0530 Subject: [PATCH 01/18] sdk%lint: describe script globals, sort alphabetically where feasible --- contrib/build_docs.py | 10 +++++++++- contrib/git_filter.py | 11 +++++++---- contrib/lint/lint_cargo.py | 5 +++++ contrib/lint/lint_unconv.py | 4 +++- contrib/lint_all.py | 1 + contrib/zen/__init__.py | 14 +++++++++----- 6 files changed, 34 insertions(+), 11 deletions(-) diff --git a/contrib/build_docs.py b/contrib/build_docs.py index cb3d33d1..127e6bce 100755 --- a/contrib/build_docs.py +++ b/contrib/build_docs.py @@ -33,9 +33,17 @@ if TYPE_CHECKING: from collections.abc import Callable -SITE_DIR = Path("public") + +# Starting port the preview server binds to. PREVIEW_PORT = 8000 + +# Target directory for build output. +SITE_DIR = Path("public") + +# Source directory of sample crates. WASM_SAMPLES_DIR = Path("contrib/samples") + +# Matches additional assets bundled with samples. WEB_ASSET_GLOBS = ("*.js", "*.css") diff --git a/contrib/git_filter.py b/contrib/git_filter.py index 94a71866..40244c4f 100755 --- a/contrib/git_filter.py +++ b/contrib/git_filter.py @@ -47,8 +47,14 @@ root_dir, ) +# Grace period for a child process before its terminated. +_CHILD_SHUTDOWN_TIMEOUT = 5 + +# Mapping between verdict and coloring. _STATUS_COLORS = {"PASS": ANSI_GREEN, "FAIL": ANSI_RED} -_GIT = "" # set in main() + +# Path to the git binary. +_GIT = "" @dataclass @@ -98,9 +104,6 @@ def _results_table(commits: list[CommitResult]) -> str: return format_table(headers, rows, _STATUS_COLORS) -_CHILD_SHUTDOWN_TIMEOUT = 5 - - def _terminate_child( child: subprocess.Popen[bytes] | None, sig: int, ) -> None: diff --git a/contrib/lint/lint_cargo.py b/contrib/lint/lint_cargo.py index 151e8e39..2b433c5c 100755 --- a/contrib/lint/lint_cargo.py +++ b/contrib/lint/lint_cargo.py @@ -35,8 +35,10 @@ touched, ) +# Base name of this script (equivalent to argv[0]). SCRIPT = Path(__file__).stem +# Platforms the dependency graph is resolved for. TARGET_TRIPLES: tuple[str, ...] = ( "x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu", @@ -46,7 +48,10 @@ # the version (source, ` (*)` dedupe marker, feature list) is ignored. TREE_ENTRY = re.compile(r"^(\S+) v(\S+)") +# A package, named by its crate and the version resolved for it. Coord = tuple[str, str] + +# A semantic version, held as its components. Version = tuple[int, ...] diff --git a/contrib/lint/lint_unconv.py b/contrib/lint/lint_unconv.py index ea5e2efa..a19a679f 100755 --- a/contrib/lint/lint_unconv.py +++ b/contrib/lint/lint_unconv.py @@ -18,9 +18,10 @@ from common import declare_verbs +# File in the repository root the accepted vocabulary is read from. CONFIG_FILENAME = "unconv.toml" -# namespace%type[(scope)][!]: description +# Matches a commit subject, 'namespace%type[(scope)][!]: description'. _PATTERN = re.compile( r"^(?P[^\s%]+)" r"%(?P[A-Za-z0-9_-]+)" @@ -29,6 +30,7 @@ r": (?P.+)$" ) +# Config keys that name a setting rather than a namespace. _RESERVED: frozenset[str] = frozenset({"global"}) diff --git a/contrib/lint_all.py b/contrib/lint_all.py index 86f16a30..83eac8c4 100755 --- a/contrib/lint_all.py +++ b/contrib/lint_all.py @@ -31,6 +31,7 @@ format_table, ) +# Colour each verdict is reported in. _STATUS_COLORS = {"pass": ANSI_GREEN, "fail": ANSI_RED} diff --git a/contrib/zen/__init__.py b/contrib/zen/__init__.py index 665ccd25..92755bb4 100644 --- a/contrib/zen/__init__.py +++ b/contrib/zen/__init__.py @@ -20,11 +20,7 @@ if TYPE_CHECKING: from markdown import Markdown -_ALERT_RE = re.compile( - r"^>[ ]?\[!(NOTE|TIP|IMPORTANT|WARNING|CAUTION)\][ ]*(.*)$" -) -_QUOTE_LINE_RE = re.compile(r"^>[ ]?(.*)$") - +# Admonition each alert kind is rendered as. _ALERT_KIND = { "NOTE": "note", "TIP": "tip", @@ -33,6 +29,14 @@ "CAUTION": "danger", } +# Matches the marker opening a GitHub-flavoured alert. +_ALERT_RE = re.compile( + r"^>[ ]?\[!(NOTE|TIP|IMPORTANT|WARNING|CAUTION)\][ ]*(.*)$" +) + +# Matches a line carrying the body of a block quote. +_QUOTE_LINE_RE = re.compile(r"^>[ ]?(.*)$") + class GfmAlertsPreprocessor(Preprocessor): """Rewrite `> [!NOTE]` blocks into admonition syntax.""" From 5e490c76e6ee85b2de97962704c7a49e0a1f2de3 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:50:08 +0530 Subject: [PATCH 02/18] sdk%refac(lint): consolidate more duplicated logic to `common.py` --- contrib/common.py | 61 ++++++++++++++++++++++++----------- contrib/git_filter.py | 47 ++++++--------------------- contrib/lint/lint_cargo.py | 18 +++++------ contrib/lint/lint_codeql.py | 6 ++-- contrib/lint/lint_markdown.py | 9 ++---- contrib/lint/lint_semgrep.py | 6 ++-- 6 files changed, 69 insertions(+), 78 deletions(-) diff --git a/contrib/common.py b/contrib/common.py index 60d208ed..3d17c7c3 100644 --- a/contrib/common.py +++ b/contrib/common.py @@ -21,6 +21,7 @@ if TYPE_CHECKING: from collections.abc import Callable, Mapping + from typing import TextIO # ANSI escape codes for terminal output. ANSI_BOLD = "\033[1m" @@ -32,6 +33,9 @@ # Cargo workspace roots, relative to the repository root. CARGO_WORKSPACES: tuple[str, ...] = (".", "contrib/samples") +# Rust source roots the analysers scan, relative to the repository root. +SOURCE_DIRS: tuple[str, ...] = ("pkgs", "contrib/samples") + # Assumed base branch for codebase. DEFAULT_BASE = "develop" @@ -96,27 +100,48 @@ def is_plain_file(root: Path, name: str) -> bool: return path.resolve().is_relative_to(root.resolve()) +def git_run(cwd: Path | str, *args: str) -> subprocess.CompletedProcess[str]: + """Run a git command in *cwd* and return the result.""" + return subprocess.run( # noqa: S603 + [require_bin("git"), *args], + capture_output=True, + check=False, + cwd=str(cwd), + encoding="utf-8", + errors="replace", + ) + + +def git_out(cwd: Path | str, *args: str) -> str: + """Run a git command in *cwd*, raise on failure, return its output.""" + result = git_run(cwd, *args) + if result.returncode != 0: + fault = result.stderr.strip() or result.stdout.strip() + raise RuntimeError(f"git {args[0]}: {fault or result.returncode}") + return result.stdout.strip() + + +def relay( + text: str, + repo_root: Path, + *, + stream: TextIO | None = None, + drop: Callable[[str], bool] | None = None, +) -> None: + """Print *text* with paths shortened against *repo_root*.""" + prefix = str(repo_root) + "/" + for line in text.splitlines(): + if drop is not None and drop(line): + continue + print(line.replace(prefix, ""), file=stream or sys.stdout) + + def touched(repo_root: Path, suffixes: tuple[str, ...]) -> list[str]: """Return the files matching *suffixes* that this branch has changed.""" - git = require_bin("git") - - def run(args: list[str]) -> str: - result = subprocess.run( # noqa: S603 - [git, *args], - capture_output=True, - check=False, - cwd=str(repo_root), - text=True, - ) - if result.returncode != 0: - raise RuntimeError( - f"git {args[0]}: {result.stderr.strip() or result.returncode}", - ) - return result.stdout - - base = run(["merge-base", DEFAULT_BASE, "HEAD"]).strip() + base = git_out(repo_root, "merge-base", DEFAULT_BASE, "HEAD") return [ - name for name in run(["diff", "--name-only", base]).splitlines() + name + for name in git_out(repo_root, "diff", "--name-only", base).splitlines() if name.endswith(suffixes) and is_plain_file(repo_root, name) ] diff --git a/contrib/git_filter.py b/contrib/git_filter.py index 40244c4f..69f5a59c 100755 --- a/contrib/git_filter.py +++ b/contrib/git_filter.py @@ -43,19 +43,17 @@ RETCODE_PASS, RETCODE_SKIP, format_table, - require_bin, + git_out, + git_run, root_dir, ) -# Grace period for a child process before its terminated. +# Grace period for a child process before it is terminated. _CHILD_SHUTDOWN_TIMEOUT = 5 # Mapping between verdict and coloring. _STATUS_COLORS = {"PASS": ANSI_GREEN, "FAIL": ANSI_RED} -# Path to the git binary. -_GIT = "" - @dataclass class CommitResult: @@ -64,32 +62,9 @@ class CommitResult: status: str = "pending" -def _git(*args: str, cwd: str) -> subprocess.CompletedProcess[str]: - """Run a git command and return the result.""" - return subprocess.run( # noqa: S603 - [_GIT, *args], - capture_output=True, - check=False, - cwd=cwd, - encoding="utf-8", - errors="replace", - ) - - -def _git_ok(*args: str, cwd: str) -> str: - """Run a git command, raise on failure, return stdout.""" - r = _git(*args, cwd=cwd) - if r.returncode != 0: - msg = r.stderr.strip() or r.stdout.strip() - raise RuntimeError(f"git {args[0]}: {msg}") - return r.stdout.strip() - - def _enumerate_commits(base: str, tip: str, cwd: str) -> list[CommitResult]: """List commits in base..tip order (oldest first).""" - out = _git_ok( - "log", "--reverse", "--format=%H%x00%s", f"{base}..{tip}", cwd=cwd, - ) + out = git_out(cwd, "log", "--reverse", "--format=%H%x00%s", f"{base}..{tip}") return [ CommitResult(*line.split("\0", 1)) for line in out.splitlines() @@ -122,7 +97,7 @@ def _terminate_child( def _remove_worktree(root: str, wt_path: str) -> None: - _git("worktree", "remove", "--force", wt_path, cwd=root) + git_run(root, "worktree", "remove", "--force", wt_path) def _parse_args() -> argparse.Namespace: @@ -165,8 +140,6 @@ def _parse_args() -> argparse.Namespace: def main() -> int: - global _GIT - _GIT = require_bin("git") args = _parse_args() root = str(root_dir()) @@ -174,8 +147,8 @@ def main() -> int: base = refs[0] if len(refs) >= 2 else DEFAULT_BASE tip = refs[-1] if refs else "HEAD" - base_hash = _git_ok("rev-parse", base, cwd=root) - tip_hash = _git_ok("rev-parse", tip, cwd=root) + base_hash = git_out(root, "rev-parse", base) + tip_hash = git_out(root, "rev-parse", tip) if base_hash == tip_hash: print(f"{base} and {tip} are identical ({base_hash[:8]})") return RETCODE_SKIP @@ -211,7 +184,7 @@ def cleanup() -> None: os.rmdir(wt_dir) atexit.register(cleanup) - _git_ok("worktree", "add", "--detach", "--quiet", wt_dir, cwd=root) + git_out(root, "worktree", "add", "--detach", "--quiet", wt_dir) wt_ready = True child: subprocess.Popen[bytes] | None = None prev_handlers = {} @@ -243,8 +216,8 @@ def handler(_signum: int, _frame: object, *, s: int = sig) -> None: failed = False try: for cr in commits: - _git_ok("checkout", "--quiet", "--force", cr.hash, cwd=wt_dir) - _git_ok("clean", "-fdx", cwd=wt_dir) + git_out(wt_dir, "checkout", "--quiet", "--force", cr.hash) + git_out(wt_dir, "clean", "-fdx") print(f"--- {cr.hash[:8]} {cr.subject} ---") child = subprocess.Popen( # noqa: S603 args.exec_cmd, diff --git a/contrib/lint/lint_cargo.py b/contrib/lint/lint_cargo.py index 2b433c5c..d165a13f 100755 --- a/contrib/lint/lint_cargo.py +++ b/contrib/lint/lint_cargo.py @@ -30,6 +30,7 @@ RETCODE_SKIP, declare_verbs, format_table, + relay, require_bin, root_dir, touched, @@ -80,15 +81,16 @@ def _check_format( cwd=str(repo_root), text=True, ) - prefix = str(repo_root) + "/" - for line in result.stdout.splitlines(): - print(line.replace(prefix, "")) + relay(result.stdout, repo_root) # Taplo reports the file count on stderr at INFO, so only the lines that # name a fault should be emitted. - for line in result.stderr.splitlines(): - if not line.lstrip().startswith("INFO"): - print(line.replace(prefix, ""), file=sys.stderr) + relay( + result.stderr, + repo_root, + stream=sys.stderr, + drop=lambda line: line.lstrip().startswith("INFO"), + ) if result.returncode != 0: if not fix: @@ -144,9 +146,7 @@ def _cargo(cargo_bin: str, repo_root: Path, args: list[str]) -> str: cwd=str(repo_root), text=True, ) - prefix = str(repo_root) + "/" - for line in result.stderr.splitlines(): - print(line.replace(prefix, ""), file=sys.stderr) + relay(result.stderr, repo_root, stream=sys.stderr) if result.returncode != 0: raise RuntimeError(f"cargo {args[0]} failed with {result.returncode}") return result.stdout diff --git a/contrib/lint/lint_codeql.py b/contrib/lint/lint_codeql.py index 251a079a..5cac7a69 100755 --- a/contrib/lint/lint_codeql.py +++ b/contrib/lint/lint_codeql.py @@ -30,6 +30,7 @@ RETCODE_ERR, RETCODE_PASS, RETCODE_SKIP, + SOURCE_DIRS, declare_verbs, require_bin, root_dir, @@ -229,10 +230,7 @@ def main(argv: list[str] | None = None) -> int: raise FileNotFoundError("no .ql queries found in contrib/codeql/") # Generate source-line data for queries that need raw text. - source_dirs = [ - repo_root / "pkgs", - repo_root / "contrib" / "samples", - ] + source_dirs = [repo_root / where for where in SOURCE_DIRS] generated = _generate_source_lines(repo_root, source_dirs, query_dir) subprocess.run( # noqa: S603 [codeql_bin, "query", "format", "-i", str(generated)], diff --git a/contrib/lint/lint_markdown.py b/contrib/lint/lint_markdown.py index 6169f097..8fb85ab6 100755 --- a/contrib/lint/lint_markdown.py +++ b/contrib/lint/lint_markdown.py @@ -14,7 +14,7 @@ import subprocess import sys -from common import RETCODE_ERR, require_bin, root_dir +from common import RETCODE_ERR, relay, require_bin, root_dir DISABLED_RULES = "md025,md033,md041" @@ -38,11 +38,8 @@ def main() -> int: text=True, ) - prefix = str(repo_root) + "/" - for line in result.stdout.splitlines(): - print(line.replace(prefix, "")) - for line in result.stderr.splitlines(): - print(line.replace(prefix, ""), file=sys.stderr) + relay(result.stdout, repo_root) + relay(result.stderr, repo_root, stream=sys.stderr) return result.returncode diff --git a/contrib/lint/lint_semgrep.py b/contrib/lint/lint_semgrep.py index 5b11878b..b25b5bc6 100755 --- a/contrib/lint/lint_semgrep.py +++ b/contrib/lint/lint_semgrep.py @@ -17,6 +17,7 @@ from common import ( RETCODE_ERR, RETCODE_PASS, + SOURCE_DIRS, require_bin, root_dir, ) @@ -27,10 +28,7 @@ def main() -> int: repo_root = root_dir() config_dir = repo_root / "contrib" / "semgrep" - target_dirs = [ - repo_root / "pkgs", - repo_root / "contrib" / "samples", - ] + target_dirs = [repo_root / where for where in SOURCE_DIRS] configs: list[str] = [] for cfg in sorted(config_dir.glob("*.yml")): From 99eda586696ecc93b41b26c6fdd342c05bcf60f2 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:43:28 +0530 Subject: [PATCH 03/18] sdk%fix(lint): anchor `lint_unconv` to the repository, not the caller --- contrib/common.py | 11 +++++++ contrib/lint/lint_unconv.py | 58 ++++++++++++------------------------- 2 files changed, 29 insertions(+), 40 deletions(-) diff --git a/contrib/common.py b/contrib/common.py index 3d17c7c3..a087af50 100644 --- a/contrib/common.py +++ b/contrib/common.py @@ -16,6 +16,7 @@ import shutil import subprocess import sys +from functools import cache from pathlib import Path from typing import TYPE_CHECKING, NoReturn @@ -190,6 +191,15 @@ def find_up( raise FileNotFoundError(f"{label} not found above {start}") +def find_up_file(start: Path, name: str) -> Path | None: + """Walk upward from *start*, returning the first *name* found.""" + for directory in (start, *start.parents): + candidate = directory / name + if candidate.is_file(): + return candidate + return None + + def is_workspace_root(d: Path) -> bool: """Return True if *d* looks like a Cargo workspace root.""" cargo = d / "Cargo.toml" @@ -211,6 +221,7 @@ def require_bin(name: str, path: str | None = None) -> str: return result +@cache def root_dir() -> Path: """Return the workspace root (directory containing Cargo.toml).""" return find_up( diff --git a/contrib/lint/lint_unconv.py b/contrib/lint/lint_unconv.py index a19a679f..142cc417 100755 --- a/contrib/lint/lint_unconv.py +++ b/contrib/lint/lint_unconv.py @@ -10,13 +10,19 @@ from __future__ import annotations import re -import subprocess import sys import tomllib from dataclasses import dataclass, field from pathlib import Path -from common import declare_verbs +from common import ( + DEFAULT_BASE, + declare_verbs, + find_up_file, + git_out, + git_run, + root_dir, +) # File in the repository root the accepted vocabulary is read from. CONFIG_FILENAME = "unconv.toml" @@ -66,15 +72,6 @@ def load(cls, path: Path) -> Config: return cls(global_types=global_types, namespaces=namespaces) -def _find_config(start: Path) -> Path | None: - """Walk upward from *start* looking for CONFIG_FILENAME.""" - for directory in (start, *start.parents): - candidate = directory / CONFIG_FILENAME - if candidate.is_file(): - return candidate - return None - - def _allowed_types(namespace: str, config: Config) -> frozenset[str]: ns = config.namespaces.get(namespace) if ns is None: @@ -125,54 +122,35 @@ def _lint_subject(subject: str, config: Config) -> list[str]: def _current_branch() -> str | None: - result = subprocess.run( - ["git", "rev-parse", "--abbrev-ref", "HEAD"], # noqa: S607 - capture_output=True, - check=False, - text=True, - ) + result = git_run(root_dir(), "rev-parse", "--abbrev-ref", "HEAD") if result.returncode != 0: return None return result.stdout.strip() def _ref_exists(ref: str) -> bool: - result = subprocess.run( # noqa: S603 - ["git", "rev-parse", "--verify", "--quiet", ref], # noqa: S607 - capture_output=True, - check=False, - ) + result = git_run(root_dir(), "rev-parse", "--verify", "--quiet", ref) return result.returncode == 0 def _default_range() -> str: branch = _current_branch() - if branch is None or branch == "HEAD" or branch == "develop": + if branch is None or branch == "HEAD" or branch == DEFAULT_BASE: return "HEAD~1..HEAD" - for ref in ("develop", "origin/develop"): + for ref in (DEFAULT_BASE, f"origin/{DEFAULT_BASE}"): if _ref_exists(ref): return f"{ref}..HEAD" return "HEAD~1..HEAD" def _subjects_from_commit(ref: str) -> list[str]: - result = subprocess.run( # noqa: S603 - ["git", "log", "-1", "--format=%s", ref], # noqa: S607 - capture_output=True, - check=True, - text=True, - ) - return [line for line in result.stdout.splitlines() if line.strip()] + out = git_out(root_dir(), "log", "-1", "--format=%s", ref) + return [line for line in out.splitlines() if line.strip()] def _subjects_from_range(git_range: str) -> list[str]: - result = subprocess.run( # noqa: S603 - ["git", "log", "--format=%s", git_range], # noqa: S607 - capture_output=True, - check=True, - text=True, - ) - return [line for line in result.stdout.splitlines() if line.strip()] + out = git_out(root_dir(), "log", "--format=%s", git_range) + return [line for line in out.splitlines() if line.strip()] def main() -> int: @@ -207,10 +185,10 @@ def main() -> int: config_path: Path | None = args.f if config_path is None: - config_path = _find_config(Path.cwd()) + config_path = find_up_file(root_dir(), CONFIG_FILENAME) if config_path is None: print( - f"error: {CONFIG_FILENAME} not found (searched from {Path.cwd()})", + f"error: {CONFIG_FILENAME} not found (searched from {root_dir()})", file=sys.stderr, ) return 2 From f1e4699f9b600aa61c2876dc77578782411fad99 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:00:17 +0530 Subject: [PATCH 04/18] sdk%refac: re-home the Zensical toolchain to `docs/` --- .github/workflows/pages.yml | 8 +++---- .gitignore | 2 +- contrib/__init__.py | 1 - docs/README.md | 6 ++--- {contrib => docs}/build_docs.py | 24 ++++++++++++------- docs/common.py | 1 + contrib/zen/__init__.py => docs/preprocess.py | 2 +- zensical.toml => docs/zensical.toml | 6 ++--- pyproject.toml | 2 +- 9 files changed, 29 insertions(+), 23 deletions(-) delete mode 100644 contrib/__init__.py rename {contrib => docs}/build_docs.py (92%) create mode 120000 docs/common.py rename contrib/zen/__init__.py => docs/preprocess.py (96%) rename zensical.toml => docs/zensical.toml (96%) diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index b3a6570c..9c06eab3 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -7,9 +7,7 @@ on: paths: - pkgs/** - contrib/samples/** - - contrib/zen/** - - contrib/build_docs.py - - docs/zen/** + - docs/** - .github/workflows/pages.yml workflow_dispatch: @@ -59,12 +57,12 @@ jobs: restore-keys: cargo-deps- - name: Build documentation - run: python contrib/build_docs.py build + run: python docs/build_docs.py build - name: Upload Pages artifact uses: actions/upload-pages-artifact@v5 with: - path: public + path: docs/.site deploy: name: Deploy to GitHub Pages diff --git a/.gitignore b/.gitignore index 7f7311b4..9211f2c3 100644 --- a/.gitignore +++ b/.gitignore @@ -187,7 +187,7 @@ cython_debug/ **/e2e/ # Built site -public/ +docs/.site/ # WASM builds *.wasm diff --git a/contrib/__init__.py b/contrib/__init__.py deleted file mode 100644 index fc53dd38..00000000 --- a/contrib/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Contributor tooling packages.""" diff --git a/docs/README.md b/docs/README.md index 3fa0d709..c255a8d4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,7 +6,7 @@ in the codebase (e.g. [`pkgs/`](../pkgs/)). ## User Guide The user guide is generated using [Zensical](https://pypi.org/project/zensical/) (a fork of -[MkDocs](https://pypi.org/project/mkdocs/)), configured using [`zensical.toml`](../zensical.toml) with the documentation +[MkDocs](https://pypi.org/project/mkdocs/)), configured using [`zensical.toml`](./zensical.toml) with the documentation located in [`docs/zen`](./zen). ### Dependencies @@ -21,7 +21,7 @@ Most dependencies can be installed using `python -m pip install -e '.[dev]'`. ### Preview ```sh -python contrib/build_docs.py preview +python docs/build_docs.py preview ``` ### Building @@ -29,5 +29,5 @@ python contrib/build_docs.py preview From repository root ```sh -python contrib/build_docs.py build +python docs/build_docs.py build ``` diff --git a/contrib/build_docs.py b/docs/build_docs.py similarity index 92% rename from contrib/build_docs.py rename to docs/build_docs.py index 127e6bce..a6c2a723 100755 --- a/contrib/build_docs.py +++ b/docs/build_docs.py @@ -12,6 +12,7 @@ from __future__ import annotations import http.server +import os import re import shutil import socket @@ -33,12 +34,17 @@ if TYPE_CHECKING: from collections.abc import Callable +# Parent directory sourced by walking back from current file. +DOCS_DIR = Path(__file__).resolve().parent + +# Path to Zensical's configuration file. +CONFIG_FILE = DOCS_DIR / "zensical.toml" # Starting port the preview server binds to. PREVIEW_PORT = 8000 # Target directory for build output. -SITE_DIR = Path("public") +SITE_DIR = DOCS_DIR / ".site" # Source directory of sample crates. WASM_SAMPLES_DIR = Path("contrib/samples") @@ -54,7 +60,6 @@ def _build_wasm_samples(root: Path, wasm_pack: str) -> None: print("no WASM samples found", file=sys.stderr) return - import os import tomllib toolchain_file = root / "rust-toolchain.toml" @@ -89,16 +94,19 @@ def _build_wasm_samples(root: Path, wasm_pack: str) -> None: def _build_site(root: Path, zensical: str) -> None: """Run zensical to build the documentation site.""" + env = {**os.environ, "PYTHONPATH": str(DOCS_DIR)} subprocess.run( # noqa: S603 - [zensical, "build", "-f", str(root / "zensical.toml")], + [zensical, "build", "-f", str(CONFIG_FILE)], check=True, + cwd=str(root), + env=env, ) def _copy_artifacts(root: Path) -> None: """Copy WASM packages and web assets into the built site.""" samples = sorted((root / WASM_SAMPLES_DIR).glob("*/Cargo.toml")) - site = root / SITE_DIR + site = SITE_DIR common_css = root / WASM_SAMPLES_DIR / "common.css" if common_css.is_file(): @@ -177,8 +185,8 @@ def _build(root: Path) -> None: _build_wasm_samples(root, wasm_pack) _build_site(root, zensical) _copy_artifacts(root) - _generate_pygments_css(root / SITE_DIR) - _minify_js(root / SITE_DIR) + _generate_pygments_css(SITE_DIR) + _minify_js(SITE_DIR) def _find_free_port(host: str, start: int) -> int: @@ -196,7 +204,7 @@ def _find_free_port(host: str, start: int) -> int: def _preview(root: Path) -> None: """Build then serve the site on localhost for testing.""" _build(root) - site = root / SITE_DIR + site = SITE_DIR handler = partial( http.server.SimpleHTTPRequestHandler, @@ -217,7 +225,7 @@ def _preview(root: Path) -> None: VERBS: dict[str, tuple[Callable[[Path], None], str]] = { - "build": (_build, f"render the site into {SITE_DIR}/"), + "build": (_build, f"render the site into {SITE_DIR.name}/"), "preview": (_preview, "render the site, then serve it over localhost"), } diff --git a/docs/common.py b/docs/common.py new file mode 120000 index 00000000..7ddd8eec --- /dev/null +++ b/docs/common.py @@ -0,0 +1 @@ +../contrib/common.py \ No newline at end of file diff --git a/contrib/zen/__init__.py b/docs/preprocess.py similarity index 96% rename from contrib/zen/__init__.py rename to docs/preprocess.py index 92755bb4..8f6caff7 100644 --- a/contrib/zen/__init__.py +++ b/docs/preprocess.py @@ -87,7 +87,7 @@ class GfmAlertsExtension(Extension): """Markdown extension entrypoint for alert rewriting.""" def extendMarkdown(self, md: Markdown) -> None: - md.preprocessors.register(GfmAlertsPreprocessor(md), "zen", 110) + md.preprocessors.register(GfmAlertsPreprocessor(md), "gfm_alerts", 110) def makeExtension(**kwargs: object) -> GfmAlertsExtension: diff --git a/zensical.toml b/docs/zensical.toml similarity index 96% rename from zensical.toml rename to docs/zensical.toml index b4a51a34..564e82e8 100644 --- a/zensical.toml +++ b/docs/zensical.toml @@ -3,8 +3,8 @@ site_name = "Base SDK for Dash" site_description = "Documentation for the Base SDK for Dash." site_author = "The Dash Core developers" copyright = "Copyright © 2026-present, The Dash Core developers" -docs_dir = "docs/zen" -site_dir = "public" +docs_dir = "zen" +site_dir = ".site" use_directory_urls = true repo_url = "https://github.com/dashpay/base-sdk" repo_name = "dashpay/base-sdk" @@ -55,10 +55,10 @@ toggle.name = "Switch to light mode" [project.markdown_extensions.abbr] [project.markdown_extensions.admonition] [project.markdown_extensions.attr_list] -[project.markdown_extensions."contrib.zen"] [project.markdown_extensions.def_list] [project.markdown_extensions.footnotes] [project.markdown_extensions.md_in_html] +[project.markdown_extensions.preprocess] [project.markdown_extensions.toc] permalink = true [project.markdown_extensions.pymdownx.betterem] diff --git a/pyproject.toml b/pyproject.toml index d8e532ce..f36ff598 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.setuptools] -packages = ["contrib", "contrib.zen"] +packages = [] [tool.ruff] indent-width = 2 From 3e091b28031f840e9d3efe9e7e8227ec5bc56b60 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:54:01 +0530 Subject: [PATCH 05/18] sdk%fix(zen): abort the site build on Zensical warnings --- docs/build_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/build_docs.py b/docs/build_docs.py index a6c2a723..0b07a6ba 100755 --- a/docs/build_docs.py +++ b/docs/build_docs.py @@ -96,7 +96,7 @@ def _build_site(root: Path, zensical: str) -> None: """Run zensical to build the documentation site.""" env = {**os.environ, "PYTHONPATH": str(DOCS_DIR)} subprocess.run( # noqa: S603 - [zensical, "build", "-f", str(CONFIG_FILE)], + [zensical, "build", "--strict", "-f", str(CONFIG_FILE)], check=True, cwd=str(root), env=env, From 35f08a0cb6ab4fe6400d0bfbd7815bbd9f291a45 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Tue, 1 Sep 2026 19:03:14 +0530 Subject: [PATCH 06/18] sdk%refac: flatten `docs/zen` --- CLAUDE.md | 2 +- docs/README.md | 79 +++++++++++++------ docs/dev/about_docs.md | 26 ++++++ docs/{ => dev}/guide_rust.md | 0 docs/{zen => }/logo.svg | 0 .../samples/index.md => samples/README.md} | 4 +- .../index.md => samples/parser/README.md} | 0 .../index.md => samples/solver/README.md} | 0 docs/{zen => }/style.css | 0 docs/zen/index.md | 68 ---------------- docs/zensical.toml | 12 +-- 11 files changed, 92 insertions(+), 99 deletions(-) create mode 100644 docs/dev/about_docs.md rename docs/{ => dev}/guide_rust.md (100%) rename docs/{zen => }/logo.svg (100%) rename docs/{zen/samples/index.md => samples/README.md} (60%) rename docs/{zen/samples/parser/index.md => samples/parser/README.md} (100%) rename docs/{zen/samples/solver/index.md => samples/solver/README.md} (100%) rename docs/{zen => }/style.css (100%) delete mode 100644 docs/zen/index.md diff --git a/CLAUDE.md b/CLAUDE.md index 5bc4df0a..8795eb28 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,7 +2,7 @@ ## Coding style -The full guide is at [`docs/guide_rust.md`](./docs/guide_rust.md). Key points: +The full guide is at [`docs/dev/guide_rust.md`](./docs/dev/guide_rust.md). Key points: - **Formatting**: 2-space indentation, LF line endings, no trailing whitespace, single newline at end of file. Max line width 120, comment width 80. Enforced by `rustfmt.toml`. diff --git a/docs/README.md b/docs/README.md index c255a8d4..a4380544 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,33 +1,68 @@ -## Coding Standards +# Introduction -See [`guide_rust.md`](./guide_rust.md) for guidance for new and existing contributors applicable to Rust crates -in the codebase (e.g. [`pkgs/`](../pkgs/)). +The Base SDK for Dash is a collection of packages that enable applications to parse, construct and interact with the +Dash blockchain. This is achieved by leveraging the [`rust-bitcoin`](https://github.com/rust-bitcoin/rust-bitcoin) +ecosystem to provide a framework that builds upon existing, familiar APIs and contract expectations. -## User Guide +To achieve this, state-independent portions of the consensus engine have been split into composable packages that can be +imported as needed. -The user guide is generated using [Zensical](https://pypi.org/project/zensical/) (a fork of -[MkDocs](https://pypi.org/project/mkdocs/)), configured using [`zensical.toml`](./zensical.toml) with the documentation -located in [`docs/zen`](./zen). +Some packages depend on other packages to expose their full functionality. Packages are primarily placed in three +buckets (see below). When upgrading downstream packages, it is recommended to version-match with the lowermost bucket +utilised by your packages. -### Dependencies +* Foundation packages. These packages provide unopinionated Bitcoin-compatible data manipulation. +* Base packages. These packages implement specific algorithms but without chain-distinguishing consensus logic. +* Protocol packages. These packages define the Dash protocol as deployed, blocks, transactions, chain parameters. -Most dependencies can be installed using `python -m pip install -e '.[dev]'`. +--8<-- "README.md:crate-graph" -* Zensical (included in `[dev]`) -* PyMarkdown (included in `[dev]`) -* rjsmin (included in `[dev]`) -* [wasm-pack](https://github.com/wasm-bindgen/wasm-pack) +*Note: Solid lines are build dependencies, dotted lines are test dependencies.* -### Preview +## Versioning and Platform Policy -```sh -python docs/build_docs.py preview -``` +> [!WARNING] +> +> As the Base SDK is in early development, the public contract and API shape should be considered **unstable**. Releases +> are made sporadically on an *ad hoc* basis with `YYYY-MM-DD` [tags](https://github.com/dashpay/base-sdk/tags) and +> packages have a fixed version of `0.0.0`. Stable releases will be made in accordance with +> [Semantic Versioning](https://semver.org/). -### Building +* The minimum supported Rust version has an upper cap at the version of Rust available with Debian + [`stable`](https://wiki.debian.org/DebianStable) (last updated, [`trixie`](https://wiki.debian.org/DebianTrixie)). + The current MSRV is 1.85.0 ([source](https://github.com/dashpay/base-sdk/blob/2026-04-22/.github/workflows/build_stable.yml#L34-L35)). -From repository root +* Some features rely on Rust `nightly` (notably affected is `core::simd`, see + [rust-lang/portable-simd#364](https://github.com/rust-lang/portable-simd/issues/364)), building packages with the + `full` feature will pull in these dependencies. Consuming packages built on Rust `stable` are recommended to manually + select the features desired from the package alongside `std`. -```sh -python docs/build_docs.py build -``` + * Packages are validated against the version of nightly pinned in + [`rust-toolchain.toml`](https://github.com/dashpay/base-sdk/blob/2026-04-22/rust-toolchain.toml), SDK functionality + is tested only against the pinned version. Regressions on succeeding versions are evaluated on a case-by-case basis. + +* `no_std` + `alloc` is the baseline target, the public contract is limited by what is feasible by this baseline target. + `std` is primarily a passthrough to `std` features provided by dependent crates, no additional functionality is + offered by the SDK *itself* through `std` enablement. + +Crates by default offer the following features, any additional features offered by crates are defined in their +respective sections. + +| Feature | Capabilities enabled | Defined by every package | +| ----------- | ------------------------------------------- | --------------------------------- | +| _(baseline)_ | `no_std` + `alloc`, always available. | Yes | +| `std` | Standard library support. | Yes | +| `full` | All non-conflicting features. | Yes | +| `serde` | Serialization using `serde`. | **No** (dependent on feature set) | + +## Is the SDK a full node? + +The Base SDK is not a full node implementation nor does it intend to be. `no_std` + `alloc` restricts I/O and networking +making it better suited for resource-constrained, sandboxed or otherwise limited environments. The public contract +reflects this, restricted to only stateless or relatively context-independent portions of the overall consensus engine. + +For a more batteries-included solution, consider [`rust-dashcore`](https://github.com/dashpay/rust-dashcore) or the C++ +reference implementation, [Dash Core](https://github.com/dashpay/dash) over +[RPC](https://docs.dash.org/en/stable/docs/core/api/remote-procedure-calls.html), +[REST](https://docs.dash.org/en/stable/docs/core/api/http-rest.html) or +[ZMQ](https://docs.dash.org/en/stable/docs/core/api/zmq.html). diff --git a/docs/dev/about_docs.md b/docs/dev/about_docs.md new file mode 100644 index 00000000..5e3c2d38 --- /dev/null +++ b/docs/dev/about_docs.md @@ -0,0 +1,26 @@ +# Building Docs + +The guide is generated using [Zensical](https://pypi.org/project/zensical/) (a fork of +[MkDocs](https://pypi.org/project/mkdocs/)), configured using +[`zensical.toml`](../zensical.toml). Every source file it needs +lives in [`docs/`](..), the rendered site is written to +`docs/.site`. + +Most dependencies can be installed using `python -m pip install -e '.[dev]'`. + +* Zensical (included in `[dev]`) +* PyMarkdown (included in `[dev]`) +* rjsmin (included in `[dev]`) +* [wasm-pack](https://github.com/wasm-bindgen/wasm-pack) + +From the repository root, render the site with + +```sh +python docs/build_docs.py build +``` + +or render it and serve it over localhost with + +```sh +python docs/build_docs.py preview +``` diff --git a/docs/guide_rust.md b/docs/dev/guide_rust.md similarity index 100% rename from docs/guide_rust.md rename to docs/dev/guide_rust.md diff --git a/docs/zen/logo.svg b/docs/logo.svg similarity index 100% rename from docs/zen/logo.svg rename to docs/logo.svg diff --git a/docs/zen/samples/index.md b/docs/samples/README.md similarity index 60% rename from docs/zen/samples/index.md rename to docs/samples/README.md index ebd8bcff..32100729 100644 --- a/docs/zen/samples/index.md +++ b/docs/samples/README.md @@ -8,5 +8,5 @@ targeting the `web` platform. | Name | Description | Web-based | | ---- | ----------- | --------- | -| [Genesis Solver](solver/index.md) | Verify or mint a genesis block (uses `dash-pow`). | Yes | -| [Object Parser](parser/index.md) | Parse hex-encoded blocks or transactions into a navigable tree (uses `dash-primitives`). | Yes | +| [Genesis Solver](solver/README.md) | Verify or mint a genesis block (uses `dash-pow`). | Yes | +| [Object Parser](parser/README.md) | Parse hex-encoded blocks or transactions into a navigable tree (uses `dash-primitives`). | Yes | diff --git a/docs/zen/samples/parser/index.md b/docs/samples/parser/README.md similarity index 100% rename from docs/zen/samples/parser/index.md rename to docs/samples/parser/README.md diff --git a/docs/zen/samples/solver/index.md b/docs/samples/solver/README.md similarity index 100% rename from docs/zen/samples/solver/index.md rename to docs/samples/solver/README.md diff --git a/docs/zen/style.css b/docs/style.css similarity index 100% rename from docs/zen/style.css rename to docs/style.css diff --git a/docs/zen/index.md b/docs/zen/index.md deleted file mode 100644 index a4380544..00000000 --- a/docs/zen/index.md +++ /dev/null @@ -1,68 +0,0 @@ -# Introduction - -The Base SDK for Dash is a collection of packages that enable applications to parse, construct and interact with the -Dash blockchain. This is achieved by leveraging the [`rust-bitcoin`](https://github.com/rust-bitcoin/rust-bitcoin) -ecosystem to provide a framework that builds upon existing, familiar APIs and contract expectations. - -To achieve this, state-independent portions of the consensus engine have been split into composable packages that can be -imported as needed. - -Some packages depend on other packages to expose their full functionality. Packages are primarily placed in three -buckets (see below). When upgrading downstream packages, it is recommended to version-match with the lowermost bucket -utilised by your packages. - -* Foundation packages. These packages provide unopinionated Bitcoin-compatible data manipulation. -* Base packages. These packages implement specific algorithms but without chain-distinguishing consensus logic. -* Protocol packages. These packages define the Dash protocol as deployed, blocks, transactions, chain parameters. - ---8<-- "README.md:crate-graph" - -*Note: Solid lines are build dependencies, dotted lines are test dependencies.* - -## Versioning and Platform Policy - -> [!WARNING] -> -> As the Base SDK is in early development, the public contract and API shape should be considered **unstable**. Releases -> are made sporadically on an *ad hoc* basis with `YYYY-MM-DD` [tags](https://github.com/dashpay/base-sdk/tags) and -> packages have a fixed version of `0.0.0`. Stable releases will be made in accordance with -> [Semantic Versioning](https://semver.org/). - -* The minimum supported Rust version has an upper cap at the version of Rust available with Debian - [`stable`](https://wiki.debian.org/DebianStable) (last updated, [`trixie`](https://wiki.debian.org/DebianTrixie)). - The current MSRV is 1.85.0 ([source](https://github.com/dashpay/base-sdk/blob/2026-04-22/.github/workflows/build_stable.yml#L34-L35)). - -* Some features rely on Rust `nightly` (notably affected is `core::simd`, see - [rust-lang/portable-simd#364](https://github.com/rust-lang/portable-simd/issues/364)), building packages with the - `full` feature will pull in these dependencies. Consuming packages built on Rust `stable` are recommended to manually - select the features desired from the package alongside `std`. - - * Packages are validated against the version of nightly pinned in - [`rust-toolchain.toml`](https://github.com/dashpay/base-sdk/blob/2026-04-22/rust-toolchain.toml), SDK functionality - is tested only against the pinned version. Regressions on succeeding versions are evaluated on a case-by-case basis. - -* `no_std` + `alloc` is the baseline target, the public contract is limited by what is feasible by this baseline target. - `std` is primarily a passthrough to `std` features provided by dependent crates, no additional functionality is - offered by the SDK *itself* through `std` enablement. - -Crates by default offer the following features, any additional features offered by crates are defined in their -respective sections. - -| Feature | Capabilities enabled | Defined by every package | -| ----------- | ------------------------------------------- | --------------------------------- | -| _(baseline)_ | `no_std` + `alloc`, always available. | Yes | -| `std` | Standard library support. | Yes | -| `full` | All non-conflicting features. | Yes | -| `serde` | Serialization using `serde`. | **No** (dependent on feature set) | - -## Is the SDK a full node? - -The Base SDK is not a full node implementation nor does it intend to be. `no_std` + `alloc` restricts I/O and networking -making it better suited for resource-constrained, sandboxed or otherwise limited environments. The public contract -reflects this, restricted to only stateless or relatively context-independent portions of the overall consensus engine. - -For a more batteries-included solution, consider [`rust-dashcore`](https://github.com/dashpay/rust-dashcore) or the C++ -reference implementation, [Dash Core](https://github.com/dashpay/dash) over -[RPC](https://docs.dash.org/en/stable/docs/core/api/remote-procedure-calls.html), -[REST](https://docs.dash.org/en/stable/docs/core/api/http-rest.html) or -[ZMQ](https://docs.dash.org/en/stable/docs/core/api/zmq.html). diff --git a/docs/zensical.toml b/docs/zensical.toml index 564e82e8..3e5891db 100644 --- a/docs/zensical.toml +++ b/docs/zensical.toml @@ -3,19 +3,19 @@ site_name = "Base SDK for Dash" site_description = "Documentation for the Base SDK for Dash." site_author = "The Dash Core developers" copyright = "Copyright © 2026-present, The Dash Core developers" -docs_dir = "zen" +docs_dir = "." site_dir = ".site" use_directory_urls = true repo_url = "https://github.com/dashpay/base-sdk" repo_name = "dashpay/base-sdk" -edit_uri = "edit/develop/docs/zen" +edit_uri = "edit/develop/docs" extra_css = ["style.css"] nav = [ - { "Home" = "index.md" }, + { "Home" = "README.md" }, { "Samples" = [ - { "Samples" = "samples/index.md" }, - { "Genesis Solver" = "samples/solver/index.md" }, - { "Object Parser" = "samples/parser/index.md" }, + { "Samples" = "samples/README.md" }, + { "Genesis Solver" = "samples/solver/README.md" }, + { "Object Parser" = "samples/parser/README.md" }, ] }, ] From 2d0fa3286555b3d8b2f57b24f2ebfd00a77d0cbf Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:34:30 +0530 Subject: [PATCH 07/18] sdk%feat(zen): trim contents from site output with `.zenignore` --- docs/.zenignore | 5 +++++ docs/build_docs.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 docs/.zenignore diff --git a/docs/.zenignore b/docs/.zenignore new file mode 100644 index 00000000..93233bc0 --- /dev/null +++ b/docs/.zenignore @@ -0,0 +1,5 @@ +/*.py +/.zenignore +/zensical.toml +.DS_Store +__pycache__/ diff --git a/docs/build_docs.py b/docs/build_docs.py index 0b07a6ba..34d393cd 100755 --- a/docs/build_docs.py +++ b/docs/build_docs.py @@ -40,6 +40,9 @@ # Path to Zensical's configuration file. CONFIG_FILE = DOCS_DIR / "zensical.toml" +# Contents to drop from the site once Zensical has copied it in. +IGNORE_FILE = DOCS_DIR / ".zenignore" + # Starting port the preview server binds to. PREVIEW_PORT = 8000 @@ -136,6 +139,37 @@ def _copy_artifacts(root: Path) -> None: shutil.copy2(asset, dest) +def _parse_ignorelist() -> list[str]: + """Translate *IGNORE_FILE* into globs rooted at the built site.""" + globs = [] + lines = IGNORE_FILE.read_text(encoding="utf-8").splitlines() + for number, line in enumerate(lines, start=1): + entry = line.strip() + if not entry or entry.startswith("#"): + continue + if entry.startswith("!"): + where = f"{IGNORE_FILE.name}:{number}" + raise ValueError(f"{where}: negation is not supported") + if entry.startswith("/"): + globs.append(entry.removeprefix("/")) + elif "/" in entry.rstrip("/"): + globs.append(entry) + else: + globs.append(f"**/{entry}") + return globs + + +def _trim_site(site: Path) -> None: + """Trim files that are supposed to be excluded from site bundle.""" + for pattern in _parse_ignorelist(): + for stale in sorted(site.glob(pattern)): + print(f"pruning {stale}") + if stale.is_dir(): + shutil.rmtree(stale) + else: + stale.unlink() + + def _generate_pygments_css(site: Path) -> None: """Append Pygments syntax-highlight CSS to the built style.css.""" from pygments.formatters import HtmlFormatter @@ -185,6 +219,7 @@ def _build(root: Path) -> None: _build_wasm_samples(root, wasm_pack) _build_site(root, zensical) _copy_artifacts(root) + _trim_site(SITE_DIR) _generate_pygments_css(SITE_DIR) _minify_js(SITE_DIR) From 10d9ad21aeff9cc6071302cacf1da7219ad49fa5 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:21:26 +0530 Subject: [PATCH 08/18] sdk%refac: move WASM samples from `contrib/` to `docs/` --- .github/workflows/build_msrv.yml | 4 ++-- .github/workflows/build_nightly.yml | 2 +- .github/workflows/build_stable.yml | 2 +- .github/workflows/pages.yml | 3 +-- contrib/common.py | 4 ++-- contrib/js/eslint.config.mjs | 4 ++-- contrib/lint/lint_javascript.py | 2 +- contrib/semgrep/workspace.yml | 26 ++++++++++----------- docs/.zenignore | 7 ++++++ docs/build_docs.py | 17 +++++++------- {contrib => docs}/samples/Cargo.lock | 0 {contrib => docs}/samples/Cargo.toml | 0 {contrib => docs}/samples/common.css | 0 {contrib => docs}/samples/parser/Cargo.toml | 0 {contrib => docs}/samples/parser/index.js | 0 {contrib => docs}/samples/parser/parser.rs | 0 {contrib => docs}/samples/parser/style.css | 0 {contrib => docs}/samples/solver/Cargo.toml | 0 {contrib => docs}/samples/solver/index.js | 0 {contrib => docs}/samples/solver/solver.rs | 0 {contrib => docs}/samples/solver/style.css | 0 {contrib => docs}/samples/solver/worker.js | 0 22 files changed, 39 insertions(+), 32 deletions(-) rename {contrib => docs}/samples/Cargo.lock (100%) rename {contrib => docs}/samples/Cargo.toml (100%) rename {contrib => docs}/samples/common.css (100%) rename {contrib => docs}/samples/parser/Cargo.toml (100%) rename {contrib => docs}/samples/parser/index.js (100%) rename {contrib => docs}/samples/parser/parser.rs (100%) rename {contrib => docs}/samples/parser/style.css (100%) rename {contrib => docs}/samples/solver/Cargo.toml (100%) rename {contrib => docs}/samples/solver/index.js (100%) rename {contrib => docs}/samples/solver/solver.rs (100%) rename {contrib => docs}/samples/solver/style.css (100%) rename {contrib => docs}/samples/solver/worker.js (100%) diff --git a/.github/workflows/build_msrv.yml b/.github/workflows/build_msrv.yml index c5beb1ea..009f4aa8 100644 --- a/.github/workflows/build_msrv.yml +++ b/.github/workflows/build_msrv.yml @@ -67,7 +67,7 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git - key: cargo-deps-${{ hashFiles('Cargo.lock', 'contrib/samples/Cargo.lock') }} + key: cargo-deps-${{ hashFiles('Cargo.lock', 'docs/samples/Cargo.lock') }} restore-keys: cargo-deps- - name: Restore build artifacts @@ -130,7 +130,7 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git - key: cargo-deps-${{ hashFiles('Cargo.lock', 'contrib/samples/Cargo.lock') }} + key: cargo-deps-${{ hashFiles('Cargo.lock', 'docs/samples/Cargo.lock') }} restore-keys: cargo-deps- - name: Manage build artifacts diff --git a/.github/workflows/build_nightly.yml b/.github/workflows/build_nightly.yml index 92e314a1..0d529d4a 100644 --- a/.github/workflows/build_nightly.yml +++ b/.github/workflows/build_nightly.yml @@ -61,7 +61,7 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git - key: cargo-deps-${{ hashFiles('Cargo.lock', 'contrib/samples/Cargo.lock') }} + key: cargo-deps-${{ hashFiles('Cargo.lock', 'docs/samples/Cargo.lock') }} restore-keys: cargo-deps- - name: Manage build artifacts diff --git a/.github/workflows/build_stable.yml b/.github/workflows/build_stable.yml index f73bee01..67fd659d 100644 --- a/.github/workflows/build_stable.yml +++ b/.github/workflows/build_stable.yml @@ -62,7 +62,7 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git - key: cargo-deps-${{ hashFiles('Cargo.lock', 'contrib/samples/Cargo.lock') }} + key: cargo-deps-${{ hashFiles('Cargo.lock', 'docs/samples/Cargo.lock') }} restore-keys: cargo-deps- - name: Manage build artifacts diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 9c06eab3..64d833a8 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -6,7 +6,6 @@ on: pull_request: paths: - pkgs/** - - contrib/samples/** - docs/** - .github/workflows/pages.yml workflow_dispatch: @@ -53,7 +52,7 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git - key: cargo-deps-${{ hashFiles('Cargo.lock', 'contrib/samples/Cargo.lock') }} + key: cargo-deps-${{ hashFiles('Cargo.lock', 'docs/samples/Cargo.lock') }} restore-keys: cargo-deps- - name: Build documentation diff --git a/contrib/common.py b/contrib/common.py index a087af50..70814562 100644 --- a/contrib/common.py +++ b/contrib/common.py @@ -32,10 +32,10 @@ ANSI_RESET = "\033[0m" # Cargo workspace roots, relative to the repository root. -CARGO_WORKSPACES: tuple[str, ...] = (".", "contrib/samples") +CARGO_WORKSPACES: tuple[str, ...] = (".", "docs/samples") # Rust source roots the analysers scan, relative to the repository root. -SOURCE_DIRS: tuple[str, ...] = ("pkgs", "contrib/samples") +SOURCE_DIRS: tuple[str, ...] = ("pkgs", "docs/samples") # Assumed base branch for codebase. DEFAULT_BASE = "develop" diff --git a/contrib/js/eslint.config.mjs b/contrib/js/eslint.config.mjs index 376c4a98..514cc444 100644 --- a/contrib/js/eslint.config.mjs +++ b/contrib/js/eslint.config.mjs @@ -47,7 +47,7 @@ const sharedRules = { export default [ { - ignores: ["docs/**", "contrib/samples/**", "**/pkg/**"], + ignores: ["docs/**", "**/pkg/**"], languageOptions: { ecmaVersion: 2022, sourceType: "commonjs", @@ -62,7 +62,7 @@ export default [ rules: sharedRules, }, { - files: ["docs/**/*.js", "contrib/samples/**/*.js"], + files: ["docs/**/*.js"], ignores: ["**/pkg/**"], languageOptions: { ecmaVersion: 2022, diff --git a/contrib/lint/lint_javascript.py b/contrib/lint/lint_javascript.py index 6caf4e06..22339075 100755 --- a/contrib/lint/lint_javascript.py +++ b/contrib/lint/lint_javascript.py @@ -18,7 +18,7 @@ DEFAULT_TARGETS: tuple[str, ...] = ( ".github/scripts", - "contrib/samples", + "docs/samples", ) diff --git a/contrib/semgrep/workspace.yml b/contrib/semgrep/workspace.yml index ea12d282..e6d68d4e 100644 --- a/contrib/semgrep/workspace.yml +++ b/contrib/semgrep/workspace.yml @@ -4,7 +4,7 @@ rules: severity: ERROR languages: [rust] paths: - include: [/pkgs/**/*.rs, /contrib/samples/**/*.rs] + include: [/pkgs/**/*.rs, /docs/samples/**/*.rs] pattern-regex: |- (?xm) ^[ \t]* # line start, optional indent @@ -34,7 +34,7 @@ rules: severity: ERROR languages: [generic] paths: - include: [/pkgs/**/*.rs, /contrib/samples/**/*.rs] + include: [/pkgs/**/*.rs, /docs/samples/**/*.rs] exclude: ["**/lib.rs", "**/main.rs", "**/mod.rs"] options: generic_multiline: true @@ -50,7 +50,7 @@ rules: severity: ERROR languages: [rust] paths: - include: [/pkgs/**/*.rs, /contrib/samples/**/*.rs] + include: [/pkgs/**/*.rs, /docs/samples/**/*.rs] patterns: - pattern-regex: |- (?xm) @@ -81,7 +81,7 @@ rules: severity: ERROR languages: [rust] paths: - include: [/pkgs/**/*.rs, /contrib/samples/**/*.rs] + include: [/pkgs/**/*.rs, /docs/samples/**/*.rs] pattern-regex: 'derive\([^)]*(?\s+for\b' - id: macro-export-no-serde-cfg @@ -157,7 +157,7 @@ rules: exclude: - /pkgs/**/prelude.rs - /pkgs/**/prelude/mod.rs - - /contrib/samples/** + - /docs/samples/** pattern-regex: '\buse\s+(::)?alloc(::|\s)' - id: prelude-use-wildcards-only @@ -169,7 +169,7 @@ rules: exclude: - /pkgs/**/prelude.rs - /pkgs/**/prelude/mod.rs - - /contrib/samples/** + - /docs/samples/** pattern-regex: '\buse\s+crate::prelude::[^*]' - id: style-no-get-prefix @@ -177,7 +177,7 @@ rules: severity: ERROR languages: [rust] paths: - include: [/pkgs/**/*.rs, /contrib/samples/**/*.rs] + include: [/pkgs/**/*.rs, /docs/samples/**/*.rs] exclude: [/pkgs/**/tests/**, /pkgs/**/bench/**] patterns: - pattern-either: @@ -206,7 +206,7 @@ rules: severity: ERROR languages: [rust] paths: - include: [/pkgs/**/*.rs, /contrib/samples/**/*.rs] + include: [/pkgs/**/*.rs, /docs/samples/**/*.rs] pattern-regex: '^\s*//[/!]?\s*[-=]{4,}|^\s*//\s+--\s' - id: style-latin1-source @@ -214,7 +214,7 @@ rules: severity: ERROR languages: [rust] paths: - include: [/pkgs/**/*.rs, /contrib/samples/**/*.rs] + include: [/pkgs/**/*.rs, /docs/samples/**/*.rs] pattern-regex: '[^\x00-\xFF]' - id: typeid-no-manual-impl @@ -224,7 +224,7 @@ rules: paths: include: - /pkgs/**/*.rs - - /contrib/samples/**/*.rs + - /docs/samples/**/*.rs exclude: - /pkgs/types/marker/** pattern-regex: '\bconst[ \t]+TYPE_ID[ \t]*:[^;]*=' @@ -234,6 +234,6 @@ rules: severity: ERROR languages: [rust] paths: - include: [/pkgs/**/*.rs, /contrib/samples/**/*.rs] + include: [/pkgs/**/*.rs, /docs/samples/**/*.rs] exclude: ["**/lib.rs", "**/mod.rs"] pattern-regex: '^\s*pub\s+use\s' diff --git a/docs/.zenignore b/docs/.zenignore index 93233bc0..7105ccc8 100644 --- a/docs/.zenignore +++ b/docs/.zenignore @@ -1,5 +1,12 @@ /*.py +/.cache/ /.zenignore +/404.html /zensical.toml .DS_Store __pycache__/ +samples/**/.cargo/ +samples/**/Cargo.lock +samples/**/Cargo.toml +samples/**/target/ +samples/**/*.rs diff --git a/docs/build_docs.py b/docs/build_docs.py index 34d393cd..d08b8fdd 100755 --- a/docs/build_docs.py +++ b/docs/build_docs.py @@ -46,19 +46,19 @@ # Starting port the preview server binds to. PREVIEW_PORT = 8000 +# Source directory of sample crates. +SAMPLES_DIR = DOCS_DIR / "samples" + # Target directory for build output. SITE_DIR = DOCS_DIR / ".site" -# Source directory of sample crates. -WASM_SAMPLES_DIR = Path("contrib/samples") - # Matches additional assets bundled with samples. WEB_ASSET_GLOBS = ("*.js", "*.css") def _build_wasm_samples(root: Path, wasm_pack: str) -> None: - """Compile every WASM sample crate under *WASM_SAMPLES_DIR*.""" - samples = sorted((root / WASM_SAMPLES_DIR).glob("*/Cargo.toml")) + """Compile every WASM sample crate under *SAMPLES_DIR*.""" + samples = sorted(SAMPLES_DIR.glob("*/Cargo.toml")) if not samples: print("no WASM samples found", file=sys.stderr) return @@ -70,9 +70,10 @@ def _build_wasm_samples(root: Path, wasm_pack: str) -> None: channel = tomllib.load(f)["toolchain"]["channel"] env = { **os.environ, - "RUSTUP_TOOLCHAIN": channel, + "CARGO_TARGET_DIR": str(root / "target" / "samples"), "CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS": "-C target-feature=+simd128", + "RUSTUP_TOOLCHAIN": channel, } for cargo_toml in samples: @@ -108,10 +109,10 @@ def _build_site(root: Path, zensical: str) -> None: def _copy_artifacts(root: Path) -> None: """Copy WASM packages and web assets into the built site.""" - samples = sorted((root / WASM_SAMPLES_DIR).glob("*/Cargo.toml")) + samples = sorted(SAMPLES_DIR.glob("*/Cargo.toml")) site = SITE_DIR - common_css = root / WASM_SAMPLES_DIR / "common.css" + common_css = SAMPLES_DIR / "common.css" if common_css.is_file(): dest = site / "samples" / "common.css" dest.parent.mkdir(parents=True, exist_ok=True) diff --git a/contrib/samples/Cargo.lock b/docs/samples/Cargo.lock similarity index 100% rename from contrib/samples/Cargo.lock rename to docs/samples/Cargo.lock diff --git a/contrib/samples/Cargo.toml b/docs/samples/Cargo.toml similarity index 100% rename from contrib/samples/Cargo.toml rename to docs/samples/Cargo.toml diff --git a/contrib/samples/common.css b/docs/samples/common.css similarity index 100% rename from contrib/samples/common.css rename to docs/samples/common.css diff --git a/contrib/samples/parser/Cargo.toml b/docs/samples/parser/Cargo.toml similarity index 100% rename from contrib/samples/parser/Cargo.toml rename to docs/samples/parser/Cargo.toml diff --git a/contrib/samples/parser/index.js b/docs/samples/parser/index.js similarity index 100% rename from contrib/samples/parser/index.js rename to docs/samples/parser/index.js diff --git a/contrib/samples/parser/parser.rs b/docs/samples/parser/parser.rs similarity index 100% rename from contrib/samples/parser/parser.rs rename to docs/samples/parser/parser.rs diff --git a/contrib/samples/parser/style.css b/docs/samples/parser/style.css similarity index 100% rename from contrib/samples/parser/style.css rename to docs/samples/parser/style.css diff --git a/contrib/samples/solver/Cargo.toml b/docs/samples/solver/Cargo.toml similarity index 100% rename from contrib/samples/solver/Cargo.toml rename to docs/samples/solver/Cargo.toml diff --git a/contrib/samples/solver/index.js b/docs/samples/solver/index.js similarity index 100% rename from contrib/samples/solver/index.js rename to docs/samples/solver/index.js diff --git a/contrib/samples/solver/solver.rs b/docs/samples/solver/solver.rs similarity index 100% rename from contrib/samples/solver/solver.rs rename to docs/samples/solver/solver.rs diff --git a/contrib/samples/solver/style.css b/docs/samples/solver/style.css similarity index 100% rename from contrib/samples/solver/style.css rename to docs/samples/solver/style.css diff --git a/contrib/samples/solver/worker.js b/docs/samples/solver/worker.js similarity index 100% rename from contrib/samples/solver/worker.js rename to docs/samples/solver/worker.js From 2fbcfa2cc7cd2333c251965fe948b1fd2f70421f Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Tue, 1 Sep 2026 19:01:32 +0530 Subject: [PATCH 09/18] sdk%fix(zen): let Zensical copy `sample` assets, don't incl. cruft --- docs/build_docs.py | 39 +-------------------------------------- 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/docs/build_docs.py b/docs/build_docs.py index d08b8fdd..6ed72caf 100755 --- a/docs/build_docs.py +++ b/docs/build_docs.py @@ -52,10 +52,6 @@ # Target directory for build output. SITE_DIR = DOCS_DIR / ".site" -# Matches additional assets bundled with samples. -WEB_ASSET_GLOBS = ("*.js", "*.css") - - def _build_wasm_samples(root: Path, wasm_pack: str) -> None: """Compile every WASM sample crate under *SAMPLES_DIR*.""" samples = sorted(SAMPLES_DIR.glob("*/Cargo.toml")) @@ -89,6 +85,7 @@ def _build_wasm_samples(root: Path, wasm_pack: str) -> None: "web", "--out-dir", "pkg", + "--no-pack", "--no-default-features", ], check=True, @@ -107,39 +104,6 @@ def _build_site(root: Path, zensical: str) -> None: ) -def _copy_artifacts(root: Path) -> None: - """Copy WASM packages and web assets into the built site.""" - samples = sorted(SAMPLES_DIR.glob("*/Cargo.toml")) - site = SITE_DIR - - common_css = SAMPLES_DIR / "common.css" - if common_css.is_file(): - dest = site / "samples" / "common.css" - dest.parent.mkdir(parents=True, exist_ok=True) - print(f"copying {common_css} -> {dest}") - shutil.copy2(common_css, dest) - - for cargo_toml in samples: - crate_dir = cargo_toml.parent - name = crate_dir.name - dest_base = site / "samples" / name - - pkg_src = crate_dir / "pkg" - if pkg_src.is_dir(): - pkg_dest = dest_base / "pkg" - print(f"copying {pkg_src} -> {pkg_dest}") - if pkg_dest.exists(): - shutil.rmtree(pkg_dest) - shutil.copytree(pkg_src, pkg_dest) - - for pattern in WEB_ASSET_GLOBS: - for asset in crate_dir.glob(pattern): - dest = dest_base / asset.name - dest.parent.mkdir(parents=True, exist_ok=True) - print(f"copying {asset} -> {dest}") - shutil.copy2(asset, dest) - - def _parse_ignorelist() -> list[str]: """Translate *IGNORE_FILE* into globs rooted at the built site.""" globs = [] @@ -219,7 +183,6 @@ def _build(root: Path) -> None: _build_wasm_samples(root, wasm_pack) _build_site(root, zensical) - _copy_artifacts(root) _trim_site(SITE_DIR) _generate_pygments_css(SITE_DIR) _minify_js(SITE_DIR) From 4771d4c6fe0051de657670e1231bc0960a8b8490 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Tue, 1 Sep 2026 19:54:51 +0530 Subject: [PATCH 10/18] sdk%refac(zen): read the paths the build works on from `zensical.toml` --- docs/build_docs.py | 77 +++++++++++++++++++++++++++++----------------- docs/zensical.toml | 4 +++ 2 files changed, 52 insertions(+), 29 deletions(-) diff --git a/docs/build_docs.py b/docs/build_docs.py index 6ed72caf..08748446 100755 --- a/docs/build_docs.py +++ b/docs/build_docs.py @@ -18,6 +18,8 @@ import socket import subprocess import sys +import tomllib +from dataclasses import dataclass from functools import partial from pathlib import Path from typing import TYPE_CHECKING @@ -40,27 +42,44 @@ # Path to Zensical's configuration file. CONFIG_FILE = DOCS_DIR / "zensical.toml" -# Contents to drop from the site once Zensical has copied it in. -IGNORE_FILE = DOCS_DIR / ".zenignore" - # Starting port the preview server binds to. PREVIEW_PORT = 8000 -# Source directory of sample crates. -SAMPLES_DIR = DOCS_DIR / "samples" -# Target directory for build output. -SITE_DIR = DOCS_DIR / ".site" +@dataclass(frozen=True) +class Config: + """Settings the build is driven by, as read from *CONFIG_FILE*.""" + + # Contents to drop from the site once Zensical has copied it in. + ignore_file: Path + + # Source directory of sample crates. + samples_dir: Path + + # Target directory for build output. + site_dir: Path -def _build_wasm_samples(root: Path, wasm_pack: str) -> None: - """Compile every WASM sample crate under *SAMPLES_DIR*.""" - samples = sorted(SAMPLES_DIR.glob("*/Cargo.toml")) + @classmethod + def load(cls, path: Path) -> Config: + """Return the settings held by the file at *path*.""" + with path.open("rb") as f: + settings = tomllib.load(f) + project = settings["project"] + build = settings["build_docs"] + return cls( + ignore_file=DOCS_DIR / build["ignore_file"], + samples_dir=DOCS_DIR / build["samples_dir"], + site_dir=DOCS_DIR / project["site_dir"], + ) + + +def _build_wasm_samples(root: Path, wasm_pack: str, cfg: Config) -> None: + """Compile every WASM sample crate under the samples directory.""" + samples = sorted(cfg.samples_dir.glob("*/Cargo.toml")) if not samples: print("no WASM samples found", file=sys.stderr) return - import tomllib - toolchain_file = root / "rust-toolchain.toml" with toolchain_file.open("rb") as f: channel = tomllib.load(f)["toolchain"]["channel"] @@ -104,16 +123,16 @@ def _build_site(root: Path, zensical: str) -> None: ) -def _parse_ignorelist() -> list[str]: - """Translate *IGNORE_FILE* into globs rooted at the built site.""" +def _parse_ignorelist(ignore_file: Path) -> list[str]: + """Translate *ignore_file* into globs rooted at the built site.""" globs = [] - lines = IGNORE_FILE.read_text(encoding="utf-8").splitlines() + lines = ignore_file.read_text(encoding="utf-8").splitlines() for number, line in enumerate(lines, start=1): entry = line.strip() if not entry or entry.startswith("#"): continue if entry.startswith("!"): - where = f"{IGNORE_FILE.name}:{number}" + where = f"{ignore_file.name}:{number}" raise ValueError(f"{where}: negation is not supported") if entry.startswith("/"): globs.append(entry.removeprefix("/")) @@ -124,9 +143,9 @@ def _parse_ignorelist() -> list[str]: return globs -def _trim_site(site: Path) -> None: +def _trim_site(site: Path, ignore_file: Path) -> None: """Trim files that are supposed to be excluded from site bundle.""" - for pattern in _parse_ignorelist(): + for pattern in _parse_ignorelist(ignore_file): for stale in sorted(site.glob(pattern)): print(f"pruning {stale}") if stale.is_dir(): @@ -176,16 +195,16 @@ def _minify_js(site: Path) -> None: js.write_text(minified, encoding="utf-8") -def _build(root: Path) -> None: +def _build(root: Path, cfg: Config) -> None: """Run the full build pipeline.""" wasm_pack = require_bin("wasm-pack") zensical = require_bin("zensical") - _build_wasm_samples(root, wasm_pack) + _build_wasm_samples(root, wasm_pack, cfg) _build_site(root, zensical) - _trim_site(SITE_DIR) - _generate_pygments_css(SITE_DIR) - _minify_js(SITE_DIR) + _trim_site(cfg.site_dir, cfg.ignore_file) + _generate_pygments_css(cfg.site_dir) + _minify_js(cfg.site_dir) def _find_free_port(host: str, start: int) -> int: @@ -200,10 +219,10 @@ def _find_free_port(host: str, start: int) -> int: raise RuntimeError("no free port found") -def _preview(root: Path) -> None: +def _preview(root: Path, cfg: Config) -> None: """Build then serve the site on localhost for testing.""" - _build(root) - site = SITE_DIR + _build(root, cfg) + site = cfg.site_dir handler = partial( http.server.SimpleHTTPRequestHandler, @@ -223,8 +242,8 @@ def _preview(root: Path) -> None: srv.server_close() -VERBS: dict[str, tuple[Callable[[Path], None], str]] = { - "build": (_build, f"render the site into {SITE_DIR.name}/"), +VERBS: dict[str, tuple[Callable[[Path, Config], None], str]] = { + "build": (_build, "render the site into the directory it is configured for"), "preview": (_preview, "render the site, then serve it over localhost"), } @@ -235,7 +254,7 @@ def main() -> int: "Build the documentation site.", {verb: what for verb, (_, what) in VERBS.items()}, ).parse_args(sys.argv[1:]) - VERBS[args.verb][0](root_dir()) + VERBS[args.verb][0](root_dir(), Config.load(CONFIG_FILE)) return RETCODE_PASS diff --git a/docs/zensical.toml b/docs/zensical.toml index 3e5891db..13fa7af1 100644 --- a/docs/zensical.toml +++ b/docs/zensical.toml @@ -1,3 +1,7 @@ +[build_docs] +ignore_file = ".zenignore" +samples_dir = "samples" + [project] site_name = "Base SDK for Dash" site_description = "Documentation for the Base SDK for Dash." From b584f2fff5596de26c1608a4df940db2640c2b9c Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:40:07 +0530 Subject: [PATCH 11/18] sdk%feat(zen): replace `pymdownx.snippets` with path-aware preprocessor --- docs/preprocess.py | 54 +++++++++++++++++++++++++++++++++++++++++----- docs/zensical.toml | 2 -- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/docs/preprocess.py b/docs/preprocess.py index 8f6caff7..17893e47 100644 --- a/docs/preprocess.py +++ b/docs/preprocess.py @@ -14,6 +14,7 @@ import re from typing import TYPE_CHECKING +from common import root_dir from markdown.extensions import Extension from markdown.preprocessors import Preprocessor @@ -83,13 +84,56 @@ def run(self, lines: list[str]) -> list[str]: return output -class GfmAlertsExtension(Extension): - """Markdown extension entrypoint for alert rewriting.""" +# Matches a splice from another file, or one named section from it. +_INCLUDE_RE = re.compile(r'^\s*--8<--\s+"([^"]+)"\s*$') + + + +class IncludePreprocessor(Preprocessor): + """Splice in a file from elsewhere in the repository.""" + + def run(self, lines: list[str]) -> list[str]: + output: list[str] = [] + for line in lines: + match = _INCLUDE_RE.match(line) + if match is None: + output.append(line) + else: + output.extend(self._include(match.group(1))) + return output + + def _include(self, spec: str) -> list[str]: + name, _, section = spec.partition(":") + source = root_dir() / name + if not source.is_file(): + raise ValueError(f"{spec}: no such file in the repository") + + lines = source.read_text(encoding="utf-8").splitlines() + if section: + lines = _section(lines, section, spec) + return lines + + +def _section(lines: list[str], name: str, spec: str) -> list[str]: + """Return the lines between the markers naming *name*.""" + opener = f"[start:{name}]" + closer = f"[end:{name}]" + begin = next((i for i, text in enumerate(lines) if opener in text), None) + finish = next((i for i, text in enumerate(lines) if closer in text), None) + if begin is None or finish is None or finish < begin: + raise ValueError(f"{spec}: no such section") + return lines[begin + 1 : finish] + + +class PreprocessorHost(Extension): + """Markdown extension entrypoint.""" def extendMarkdown(self, md: Markdown) -> None: - md.preprocessors.register(GfmAlertsPreprocessor(md), "gfm_alerts", 110) + include = IncludePreprocessor(md) + md.preprocessors.register(include, "include", 32) + md.preprocessors.register(GfmAlertsPreprocessor(md), "gfm_alerts", 31) -def makeExtension(**kwargs: object) -> GfmAlertsExtension: +def makeExtension(**kwargs: object) -> PreprocessorHost: """Construct the extension.""" - return GfmAlertsExtension(**kwargs) + return PreprocessorHost(**kwargs) diff --git a/docs/zensical.toml b/docs/zensical.toml index 13fa7af1..6fce4440 100644 --- a/docs/zensical.toml +++ b/docs/zensical.toml @@ -77,8 +77,6 @@ use_pygments = true [project.markdown_extensions.pymdownx.inlinehilite] [project.markdown_extensions.pymdownx.mark] [project.markdown_extensions.pymdownx.smartsymbols] -[project.markdown_extensions.pymdownx.snippets] -base_path = ["."] [project.markdown_extensions.pymdownx.superfences] custom_fences = [ { name = "mermaid", class = "mermaid", format = "pymdownx.superfences.fence_code_format" }, From c93d85654d809b9426bcf801bcb319692bb0008a Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:40:13 +0530 Subject: [PATCH 12/18] sdk%feat(zen): resolve links against the file that holds them --- contrib/common.py | 35 ++++++++++ docs/build_docs.py | 99 +++++++++++++++++++++++++-- docs/preprocess.py | 164 ++++++++++++++++++++++++++++++++++++++++++--- docs/zensical.toml | 2 + 4 files changed, 285 insertions(+), 15 deletions(-) diff --git a/contrib/common.py b/contrib/common.py index 70814562..07f9a6ae 100644 --- a/contrib/common.py +++ b/contrib/common.py @@ -13,6 +13,7 @@ import argparse import os +import re import shutil import subprocess import sys @@ -45,6 +46,9 @@ RETCODE_PASS = 0 RETCODE_SKIP = 77 +# Matches an address that names something other than a path on disk. +_OFF_DISK_RE = re.compile(r"^(?:[A-Za-z][A-Za-z0-9+.\-]*:|//|#)") + class _VerbParser(argparse.ArgumentParser): """Parser spelling a usage fault in the harness' return codes.""" @@ -84,6 +88,37 @@ def declare_verbs( return parser +def off_disk(target: str) -> bool: + """Whether *target* addresses something other than a file on disk.""" + return _OFF_DISK_RE.match(target) is not None + + +@cache +def _entries(where: Path) -> frozenset[str]: + """Return the names *where* holds, as the filesystem spells them.""" + return frozenset(entry.name for entry in where.iterdir()) + + +def spelt_as_stored(root: Path, target: Path) -> bool: + """Whether *target* is spelt as the filesystem under *root* holds it. + + A case-insensitive filesystem resolves a misspelt path, so a wrong-case + link passes `exists()` on macOS and Windows and then serves a 404 from a + case-sensitive host. Each component is matched against its directory. + """ + # Normalising first drops the `..` a caller may have left in the path, + # which names no directory entry and so would fail the walk outright. + target = target.resolve() + if not target.is_relative_to(root): + return True + probe = root + for part in target.relative_to(root).parts: + if part not in _entries(probe): + return False + probe = probe / part + return True + + def is_plain_file(root: Path, name: str) -> bool: """Whether *name* is a regular file inside *root*, reached without links.""" path = root / name diff --git a/docs/build_docs.py b/docs/build_docs.py index 08748446..bc9a54d9 100755 --- a/docs/build_docs.py +++ b/docs/build_docs.py @@ -29,14 +29,18 @@ RETCODE_ERR, RETCODE_PASS, declare_verbs, + off_disk, require_bin, root_dir, + spelt_as_stored, ) +from preprocess import forge_url +from pygments.formatters import HtmlFormatter if TYPE_CHECKING: from collections.abc import Callable -# Parent directory sourced by walking back from current file. +# Directory this script lives in, which is the documentation root. DOCS_DIR = Path(__file__).resolve().parent # Path to Zensical's configuration file. @@ -45,6 +49,9 @@ # Starting port the preview server binds to. PREVIEW_PORT = 8000 +# Matches anything the browser has to fetch from the site itself. +_LINK_ATTR_RE = re.compile(r'(?:href|src)="([^"]+)"') + @dataclass(frozen=True) class Config: @@ -59,6 +66,15 @@ class Config: # Target directory for build output. site_dir: Path + # Documentation root the site is rendered from. + docs_dir: str + + # Repository serving whatever the site cannot. + repo_url: str + + # Branch of that repository the links resolve against. + branch: str + @classmethod def load(cls, path: Path) -> Config: """Return the settings held by the file at *path*.""" @@ -66,10 +82,14 @@ def load(cls, path: Path) -> Config: settings = tomllib.load(f) project = settings["project"] build = settings["build_docs"] + options = project["markdown_extensions"]["preprocess"] return cls( ignore_file=DOCS_DIR / build["ignore_file"], samples_dir=DOCS_DIR / build["samples_dir"], site_dir=DOCS_DIR / project["site_dir"], + docs_dir=str(project["docs_dir"]), + repo_url=str(project["repo_url"]).rstrip("/"), + branch=str(options["branch"]), ) @@ -114,7 +134,9 @@ def _build_wasm_samples(root: Path, wasm_pack: str, cfg: Config) -> None: def _build_site(root: Path, zensical: str) -> None: """Run zensical to build the documentation site.""" - env = {**os.environ, "PYTHONPATH": str(DOCS_DIR)} + inherited = os.environ.get("PYTHONPATH") + reach = [str(DOCS_DIR), *([inherited] if inherited else [])] + env = {**os.environ, "PYTHONPATH": os.pathsep.join(reach)} subprocess.run( # noqa: S603 [zensical, "build", "--strict", "-f", str(CONFIG_FILE)], check=True, @@ -146,7 +168,11 @@ def _parse_ignorelist(ignore_file: Path) -> list[str]: def _trim_site(site: Path, ignore_file: Path) -> None: """Trim files that are supposed to be excluded from site bundle.""" for pattern in _parse_ignorelist(ignore_file): - for stale in sorted(site.glob(pattern)): + matched = sorted(site.glob(pattern)) + if not matched: + print(f"warning: {ignore_file.name}: {pattern} matched nothing", + file=sys.stderr) + for stale in matched: print(f"pruning {stale}") if stale.is_dir(): shutil.rmtree(stale) @@ -156,8 +182,6 @@ def _trim_site(site: Path, ignore_file: Path) -> None: def _generate_pygments_css(site: Path) -> None: """Append Pygments syntax-highlight CSS to the built style.css.""" - from pygments.formatters import HtmlFormatter - # Lines Pygments prepends for line-numbered blocks (unused by this site). skip_re = re.compile(r"^(pre |td\.linenos |span\.linenos )") @@ -195,6 +219,70 @@ def _minify_js(site: Path) -> None: js.write_text(minified, encoding="utf-8") +def _relative_url(landing: Path, page: Path) -> str: + """Return the address of *landing*, as reached from *page*.""" + if landing.name == "index.html": + return os.path.relpath(landing.parent, page.parent) + "/" + return os.path.relpath(landing, page.parent) + + +def _repoint_links(site: Path, cfg: Config) -> None: + """Point a link that left the site at the repository instead.""" + # Only holds while the site root and the documentation root are one. + if cfg.docs_dir != ".": + raise ValueError("docs_dir must be the documentation root itself") + + broken: list[str] = [] + + for page in sorted(site.rglob("*.html")): + html = page.read_text(encoding="utf-8") + + def swap(match: re.Match[str], page: Path = page) -> str: + target = match.group(1) + if off_disk(target): + return match.group(0) + + path, mark, rest = target.partition("#") + # A page carried in from elsewhere is addressed from the site root. + home = site if path.startswith("/") else page.parent + landing = (home / path.lstrip("/")).resolve() + if landing.is_dir(): + landing = landing / "index.html" + # A case-insensitive filesystem resolves a misspelt address, which + # a case-sensitive host then serves as a 404. + if landing.exists() and not spelt_as_stored(root_dir(), landing): + broken.append(f"{page.relative_to(site)} -> {target} (wrong case)") + return match.group(0) + if landing.exists(): + if not path.startswith("/"): + return match.group(0) + # A site-root address only holds where the site is the web root, + # so anchor it to the page instead. + near = _relative_url(landing, page) + return match.group(0).replace(f'"{target}"', f'"{near}{mark}{rest}"') + + source = DOCS_DIR / os.path.relpath(landing, site) + if not source.exists(): + broken.append(f"{page.relative_to(site)} -> {target}") + return match.group(0) + if not spelt_as_stored(root_dir(), source): + broken.append(f"{page.relative_to(site)} -> {target} (wrong case)") + return match.group(0) + + url = forge_url(cfg.repo_url, cfg.branch, source.resolve()) + print(f"repointing {target} -> {url}") + return match.group(0).replace(f'"{target}"', f'"{url}{mark}{rest}"') + + patched = _LINK_ATTR_RE.sub(swap, html) + if patched != html: + page.write_text(patched, encoding="utf-8") + + if broken: + for link in broken: + print(f"broken link: {link}", file=sys.stderr) + raise ValueError(f"{len(broken)} broken links in the built site") + + def _build(root: Path, cfg: Config) -> None: """Run the full build pipeline.""" wasm_pack = require_bin("wasm-pack") @@ -205,6 +293,7 @@ def _build(root: Path, cfg: Config) -> None: _trim_site(cfg.site_dir, cfg.ignore_file) _generate_pygments_css(cfg.site_dir) _minify_js(cfg.site_dir) + _repoint_links(cfg.site_dir, cfg) def _find_free_port(host: str, start: int) -> int: diff --git a/docs/preprocess.py b/docs/preprocess.py index 17893e47..0eb4c373 100644 --- a/docs/preprocess.py +++ b/docs/preprocess.py @@ -12,9 +12,10 @@ from __future__ import annotations import re +from pathlib import Path from typing import TYPE_CHECKING -from common import root_dir +from common import off_disk, root_dir, spelt_as_stored from markdown.extensions import Extension from markdown.preprocessors import Preprocessor @@ -46,10 +47,12 @@ def run(self, lines: list[str]) -> list[str]: source = list(lines) output: list[str] = [] index = 0 + fences = _Fences() while index < len(source): line = source[index] - match = _ALERT_RE.match(line) + # An alert shown inside a fence is an example, not a callout. + match = None if fences.covers(line) else _ALERT_RE.match(line) if match is None: output.append(line) index += 1 @@ -84,34 +87,164 @@ def run(self, lines: list[str]) -> list[str]: return output +# Matches a code block, and whatever trails the marker on that line. +_FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})(.*)$") + # Matches a splice from another file, or one named section from it. _INCLUDE_RE = re.compile(r'^\s*--8<--\s+"([^"]+)"\s*$') +# Matches the target of an inline link, and any title trailing it. +_LINK_RE = re.compile( + r"\]\(\s*(<[^<>]*>|[^\s()]+)" + r"((?:\s+(?:\"[^\"]*\"|'[^']*'|\([^()]*\)))?\s*)\)" +) + +# Directory holding this extension, which is the documentation root. +_DOCS_ROOT = Path(__file__).resolve().parent + +# Stems Zensical serves as a directory's own index, matched as spelt here. +_INDEX_STEMS = frozenset({"README", "index"}) + +# How many includes may nest before the splice is called a cycle. +_DEPTH_LIMIT = 8 + + +class _Fences: + """Running fence state over a sequence of lines.""" + + def __init__(self, opener: str | None = None) -> None: + self.opener = opener + + def covers(self, line: str) -> bool: + """Whether *line* is fenced, counting the fence markers themselves.""" + marker = _FENCE_RE.match(line) + if marker is None: + return self.opener is not None + found, trailing = marker.group(1), marker.group(2) + if self.opener is None: + self.opener = found + # A closing fence carries no info string, so a marker that does is + # content the block holds rather than the end of it. + elif ( + found[0] == self.opener[0] + and len(found) >= len(self.opener) + and not trailing.strip() + ): + self.opener = None + return True + + +def forge_url(repo_url: str, branch: str, source: Path) -> str: + """Return the URL *source* is served from, by its kind.""" + kind = "tree" if source.is_dir() else "blob" + where = source.relative_to(root_dir()) + return f"{repo_url}/{kind}/{branch}/{where}" class IncludePreprocessor(Preprocessor): - """Splice in a file from elsewhere in the repository.""" + """Splice in a file and parse disk-local links. + + A link is relative to the file holding it, so a link carried in from elsewhere + in the repository cannot be served from the site and is pointed at the + upstream host instead. + """ + + def __init__(self, md: Markdown, repo_url: str, branch: str) -> None: + super().__init__(md) + self.repo_url = repo_url.rstrip("/") + self.branch = branch def run(self, lines: list[str]) -> list[str]: + return self._expand(lines, _DEPTH_LIMIT, _Fences()) + + def _expand( + self, lines: list[str], budget: int, fences: _Fences, + ) -> list[str]: + """Splice every include in *lines*, recursing *budget* levels deep. + + *fences* is shared across levels because the renderer sees one stream, + a fence opened by an included file still holds when the parent resumes. + """ output: list[str] = [] + for line in lines: - match = _INCLUDE_RE.match(line) + # A directive inside a fence is the syntax being shown, not used. + match = None if fences.covers(line) else _INCLUDE_RE.match(line) if match is None: output.append(line) - else: - output.extend(self._include(match.group(1))) + continue + if budget <= 0: + raise ValueError(f"{match.group(1)}: includes nested past the limit") + spliced = self._include(match.group(1), fences) + output.extend(self._expand(spliced, budget - 1, fences)) + return output - def _include(self, spec: str) -> list[str]: + def _include(self, spec: str, fences: _Fences) -> list[str]: name, _, section = spec.partition(":") - source = root_dir() / name + source = (root_dir() / name).resolve() + # An absolute *name* would displace the root it is joined to, so the + # containment check has to follow the join rather than precede it. + if not source.is_relative_to(root_dir()): + raise ValueError(f"{spec}: outside the repository") if not source.is_file(): raise ValueError(f"{spec}: no such file in the repository") lines = source.read_text(encoding="utf-8").splitlines() if section: lines = _section(lines, section, spec) - return lines + # `_rebase` walks these same lines, so it takes a copy of the fence + # state at the splice point rather than advancing the caller's. + return self._rebase(lines, source.parent, _Fences(fences.opener)) + + def _rebase( + self, lines: list[str], home: Path, fences: _Fences, + ) -> list[str]: + output: list[str] = [] + + for line in lines: + if fences.covers(line): + output.append(line) + else: + output.append(_LINK_RE.sub(lambda m: self._point(m, home), line)) + + return output + + def _point(self, match: re.Match[str], home: Path) -> str: + target, title = match.group(1), match.group(2) + caged = target.startswith("<") and target.endswith(">") + if caged: + target = target[1:-1] + if off_disk(target): + return match.group(0) + + path, mark, rest = target.partition("#") + # A site-root address resolves against the built site, which only + # postprocessing can see, so it is left for that pass to anchor. + if path.startswith("/"): + return match.group(0) + source = (home / path).resolve() + if not source.exists(): + raise ValueError(f"{target}: no such file, relative to {home}") + if not source.is_relative_to(root_dir()): + raise ValueError(f"{target}: outside the repository") + if not spelt_as_stored(root_dir(), source): + raise ValueError(f"{target}: not spelt as the repository holds it") + + where = f"{self._address(source)}{mark}{rest}" + return f"]({f'<{where}>' if caged else where}{title})" + + def _address(self, source: Path) -> str: + """Return where *source* is served from, preferring the site.""" + if source.suffix != ".md" or not source.is_relative_to(_DOCS_ROOT): + return forge_url(self.repo_url, self.branch, source) + + # A page is addressed from the site root, as the file the link was + # carried in from says nothing about the page it lands on. + where = [*source.relative_to(_DOCS_ROOT).parent.parts] + if source.stem not in _INDEX_STEMS: + where.append(source.stem) + return "/" + "".join(f"{part}/" for part in where) def _section(lines: list[str], name: str, spec: str) -> list[str]: @@ -128,8 +261,19 @@ def _section(lines: list[str], name: str, spec: str) -> list[str]: class PreprocessorHost(Extension): """Markdown extension entrypoint.""" + def __init__(self, **kwargs: object) -> None: + self.config = { + "repo_url": ["", "Repository the root-relative links point at"], + "branch": ["", "Branch those links resolve against"], + } + super().__init__(**kwargs) + def extendMarkdown(self, md: Markdown) -> None: - include = IncludePreprocessor(md) + include = IncludePreprocessor( + md, + str(self.getConfig("repo_url")), + str(self.getConfig("branch")), + ) md.preprocessors.register(include, "include", 32) md.preprocessors.register(GfmAlertsPreprocessor(md), "gfm_alerts", 31) diff --git a/docs/zensical.toml b/docs/zensical.toml index 6fce4440..2e100279 100644 --- a/docs/zensical.toml +++ b/docs/zensical.toml @@ -63,6 +63,8 @@ toggle.name = "Switch to light mode" [project.markdown_extensions.footnotes] [project.markdown_extensions.md_in_html] [project.markdown_extensions.preprocess] +repo_url = "https://github.com/dashpay/base-sdk" +branch = "develop" [project.markdown_extensions.toc] permalink = true [project.markdown_extensions.pymdownx.betterem] From 83b86b28f217b3f91e1f88cf150acfe2e1e6bd4f Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:41:59 +0530 Subject: [PATCH 13/18] sdk%test(zen): add test coverage for the preprocessor --- .github/workflows/pages.yml | 3 + docs/preprocess.py | 203 ++++++++++++++++++++++++++++++++++++ pyproject.toml | 9 ++ 3 files changed, 215 insertions(+) diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 64d833a8..7f681835 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -46,6 +46,9 @@ jobs: - name: Install Python dependencies run: pip install ".[dev]" + - name: Test documentation tooling + run: pytest + - name: Manage cargo registry uses: actions/cache@v5 with: diff --git a/docs/preprocess.py b/docs/preprocess.py index 0eb4c373..c4cd5c02 100644 --- a/docs/preprocess.py +++ b/docs/preprocess.py @@ -12,7 +12,9 @@ from __future__ import annotations import re +from contextlib import contextmanager from pathlib import Path +from tempfile import TemporaryDirectory from typing import TYPE_CHECKING from common import off_disk, root_dir, spelt_as_stored @@ -20,6 +22,8 @@ from markdown.preprocessors import Preprocessor if TYPE_CHECKING: + from collections.abc import Iterator + from markdown import Markdown # Admonition each alert kind is rendered as. @@ -281,3 +285,202 @@ def extendMarkdown(self, md: Markdown) -> None: def makeExtension(**kwargs: object) -> PreprocessorHost: """Construct the extension.""" return PreprocessorHost(**kwargs) + +# Stand-in forge the tests resolve their fixtures against. +_REPO = "https://forge.test/owner/repo" +_BRANCH = "trunk" + + +class TestPreprocess: + """Tests for this module, run with `pytest docs/preprocess.py`.""" + + @staticmethod + def _render(source: str) -> str: + import markdown + + return markdown.Markdown( + extensions=[ + PreprocessorHost(repo_url=_REPO, branch=_BRANCH), + "admonition", + ], + ).convert(source) + + @staticmethod + @contextmanager + def _scratch(**files: str) -> Iterator[Path]: + with TemporaryDirectory(dir=root_dir()) as name: + home = Path(name) + for stem, text in files.items(): + (home / f"{stem}.md").write_text(text, encoding="utf-8") + yield home.relative_to(root_dir()) + + def test_alert_becomes_admonition(self) -> None: + out = self._render("> [!CAUTION]\n> Mind the gap.\n") + assert 'class="admonition danger"' in out + assert "Mind the gap." in out + + def test_alert_inside_a_fence_is_left_alone(self) -> None: + out = self._render("```\n> [!NOTE]\n> Shown, not rendered.\n```\n") + assert "[!NOTE]" in out + assert "admonition" not in out + + def test_include_splices_a_section(self) -> None: + with self._scratch( + whole="above\n\ninside\n\nbelow\n", + ) as home: + out = self._render(f'--8<-- "{home}/whole.md:mid"\n') + assert "inside" in out + assert "above" not in out + assert "below" not in out + + def test_include_refuses_an_unknown_section(self) -> None: + import pytest + + with self._scratch(whole="nothing marked\n") as home: + with pytest.raises(ValueError, match="no such section"): + self._render(f'--8<-- "{home}/whole.md:mid"\n') + + def test_include_rejects_an_absolute_path(self) -> None: + import pytest + + with pytest.raises(ValueError, match="outside the repository"): + self._render('--8<-- "/etc/hosts"\n') + + def test_include_rejects_a_traversal(self) -> None: + import pytest + + with pytest.raises(ValueError, match="outside the repository"): + self._render('--8<-- "../../../../etc/hosts"\n') + + def test_include_inside_a_fence_is_left_alone(self) -> None: + out = self._render('```\n--8<-- "unconv.toml"\n```\n') + assert "8<--" in out + assert "[global]" not in out + + def test_include_nests(self) -> None: + with self._scratch( + outer='--8<-- "unconv.toml"\n', + ) as home: + out = self._render(f'--8<-- "{home}/outer.md"\n') + assert "8<--" not in out + assert "global" in out + + def test_include_refuses_a_cycle(self) -> None: + import pytest + + with self._scratch(loop="") as home: + spec = f'--8<-- "{home}/loop.md"\n' + (root_dir() / home / "loop.md").write_text(spec, encoding="utf-8") + with pytest.raises(ValueError, match="nested past the limit"): + self._render(spec) + + def test_titled_link_is_rebased(self) -> None: + with self._scratch(page='[a](../README.md "root")\n') as home: + out = self._render(f'--8<-- "{home}/page.md"\n') + assert f'href="{_REPO}/blob/{_BRANCH}/README.md"' in out + assert 'title="root"' in out + + def test_caged_link_is_rebased(self) -> None: + with self._scratch(page="[a](<../README.md>)\n") as home: + out = self._render(f'--8<-- "{home}/page.md"\n') + assert f'href="{_REPO}/blob/{_BRANCH}/README.md"' in out + + def test_bare_link_is_rebased(self) -> None: + with self._scratch(page="[a](../unconv.toml)\n") as home: + out = self._render(f'--8<-- "{home}/page.md"\n') + assert f'href="{_REPO}/blob/{_BRANCH}/unconv.toml"' in out + + def test_page_under_docs_is_addressed_from_the_site(self) -> None: + with self._scratch(page="[a](../docs/dev/guide_rust.md)\n") as home: + out = self._render(f'--8<-- "{home}/page.md"\n') + assert 'href="/dev/guide_rust/"' in out + + def test_off_disk_link_is_left_alone(self) -> None: + with self._scratch(page="[a](tel:+15551212) [b](irc://x/y)\n") as home: + out = self._render(f'--8<-- "{home}/page.md"\n') + assert 'href="tel:+15551212"' in out + assert 'href="irc://x/y"' in out + + def test_missing_link_target_is_refused(self) -> None: + import pytest + + with self._scratch(page="[a](./nope.md)\n") as home: + with pytest.raises(ValueError, match="no such file"): + self._render(f'--8<-- "{home}/page.md"\n') + + @staticmethod + def _pointer() -> IncludePreprocessor: + import markdown + + return IncludePreprocessor(markdown.Markdown(), _REPO, _BRANCH) + + def test_address_collapses_the_stems_zensical_indexes(self) -> None: + at = self._pointer() + for stem in ("README", "index"): + assert at._address(_DOCS_ROOT / "kit" / f"{stem}.md") == "/kit/" + + def test_address_keeps_the_stems_zensical_serves_as_pages(self) -> None: + at = self._pointer() + assert at._address(_DOCS_ROOT / "kit" / "readme.md") == "/kit/readme/" + assert at._address(_DOCS_ROOT / "kit" / "Index.md") == "/kit/Index/" + assert at._address(_DOCS_ROOT / "kit" / "guide.md") == "/kit/guide/" + + def test_spelling_check_matches_the_stored_name(self) -> None: + root = root_dir() + assert spelt_as_stored(root, root / "README.md") + assert not spelt_as_stored(root, root / "README.MD") + assert not spelt_as_stored(root, root / "Docs" / "README.md") + + def test_wrong_case_link_is_refused(self) -> None: + import pytest + + # Refused either as missing or as misspelt, by the host's case rules. + with self._scratch(page="[a](../README.MD)\n") as home: + with pytest.raises(ValueError, match=r"no such file|not spelt"): + self._render(f'--8<-- "{home}/page.md"\n') + + def test_include_survives_a_fenced_info_string(self) -> None: + out = self._render('```\n```text\n--8<-- "unconv.toml"\n```\n') + assert "8<--" in out + assert "[global]" not in out + + def test_alert_survives_a_fenced_info_string(self) -> None: + out = self._render("```\n```text\n> [!NOTE]\n> Shown.\n```\n") + assert "[!NOTE]" in out + assert "admonition" not in out + + def test_site_root_link_is_left_for_postprocessing(self) -> None: + with self._scratch(page="[a](/dev/about_docs/)\n") as home: + out = self._render(f'--8<-- "{home}/page.md"\n') + assert 'href="/dev/about_docs/"' in out + + def test_a_fence_an_include_opens_holds_over_the_parent(self) -> None: + with self._scratch(opener="```\n", body="spliced text\n") as home: + out = self._pointer().run([ + f'--8<-- "{home}/opener.md"', + f'--8<-- "{home}/body.md"', + "```", + f'--8<-- "{home}/body.md"', + ]) + # Held back while the fence the first splice opened is still open, + # then spliced once the parent's own marker closes that fence. + assert out[0] == "```" + assert out[1].startswith("--8<--") + assert out[2] == "```" + assert out[3] == "spliced text" + + def test_fences_ignore_a_marker_carrying_text(self) -> None: + fences = _Fences() + assert fences.covers("```") + assert fences.covers("```text") + assert fences.covers("still inside") + assert fences.covers("```") + assert not fences.covers("outside") + + def test_fences_close_only_on_a_matching_marker(self) -> None: + fences = _Fences() + assert fences.covers("````") + assert fences.covers("```") + assert fences.covers("plain text") + assert fences.covers("````") + assert not fences.covers("plain text") diff --git a/pyproject.toml b/pyproject.toml index f36ff598..3979d41d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ dev = [ "Markdown>=3.7", "pymarkdownlnt>=0.9.37", "pygments-styles>=0.3.0", + "pytest>=8", "rjsmin>=1.2", "ruff>=0.9", "semgrep>=1.118.0", @@ -19,6 +20,10 @@ dev = [ requires = ["setuptools"] build-backend = "setuptools.build_meta" +[tool.pytest.ini_options] +python_files = ["test_*.py", "*_test.py", "preprocess.py"] +testpaths = ["docs"] + [tool.setuptools] packages = [] @@ -42,6 +47,10 @@ select = [ "W", # pycodestyle warnings ] +[tool.ruff.lint.per-file-ignores] +# Suppression due to pytest module. +"docs/preprocess.py" = ["S101"] + [tool.ruff.format] indent-style = "space" line-ending = "lf" From 011b9fd6a49eb4de9332319d05c771d62ea08a1e Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:47:54 +0530 Subject: [PATCH 14/18] sdk%doc(zen): list developer docs under header `Contributing` --- docs/zensical.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/zensical.toml b/docs/zensical.toml index 2e100279..d3fd6019 100644 --- a/docs/zensical.toml +++ b/docs/zensical.toml @@ -16,6 +16,10 @@ edit_uri = "edit/develop/docs" extra_css = ["style.css"] nav = [ { "Home" = "README.md" }, + { "Contributing" = [ + { "Documentation" = "dev/about_docs.md" }, + { "Style Guide (Rust)" = "dev/guide_rust.md" }, + ] }, { "Samples" = [ { "Samples" = "samples/README.md" }, { "Genesis Solver" = "samples/solver/README.md" }, From b2589e6f1e8ac57b86b9f6f3b2d41e03c6c9435a Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:48:11 +0530 Subject: [PATCH 15/18] sdk%doc(zen): reformat style guide to be more suitable for web preview --- docs/dev/guide_rust.md | 103 +++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 65 deletions(-) diff --git a/docs/dev/guide_rust.md b/docs/dev/guide_rust.md index 54e6f4d2..2c5eccbf 100644 --- a/docs/dev/guide_rust.md +++ b/docs/dev/guide_rust.md @@ -1,25 +1,6 @@ -# Rust Development Guide +# Style Guide (Rust) -## Table of Contents - -- [Coding Style (Rust)](#coding-style-rust) - - [Formatting](#formatting) - - [Naming](#naming) - - [Type Safety](#type-safety) - - [Error Handling](#error-handling) - - [Ownership and Borrowing](#ownership-and-borrowing) - - [Conversions](#conversions) - - [Traits and Implementations](#traits-and-implementations) - - [Generics](#generics) - - [Iterators](#iterators) - - [Code Comments](#code-comments) -- [Development Guidelines](#development-guidelines) - - [Input Validation](#input-validation) - - [Security](#security) - -## Coding Style (Rust) - -### Formatting +## Formatting - Use 2-space indentation, LF line endings - Files end with a single newline @@ -30,9 +11,7 @@ Example code: ```rust -fn decode_block( - raw: &[u8], -) -> Result { +fn decode_block(raw: &[u8]) -> Result { let header = decode_header(raw)?; let txs = decode_transactions(&raw[80..])?; Ok(Block { header, txs }) @@ -41,7 +20,7 @@ fn decode_block( -### Naming +## Naming | Form | Used for | | ---------------------- | --------------------------------------------------- | @@ -61,6 +40,8 @@ fn decode_block( Example code: ```rust +const MAX_BLOCK_SIZE: usize = 2_000_000; + struct ChainTip { height: u64, hash: BlockHash, @@ -78,17 +59,19 @@ impl ChainTip { &mut self.hash } } - -const MAX_BLOCK_SIZE: usize = 2_000_000; ``` -### Type Safety +## Type Safety The type system is the first line of defence. A constraint expressed as a type is checked at compile time and costs nothing at runtime. +> [!TIP] +> Prefer `#[derive]` for standard trait implementations. A manual implementation is warranted only when the +> derived behaviour would be incorrect or when redaction is needed. + - Wrap primitive types in newtypes when two values of the same underlying type carry different semantics; this prevents accidental transposition of arguments - Prefer enums over booleans for function parameters because `Script::classify(P2pkh)` communicates intent where @@ -100,10 +83,6 @@ costs nothing at runtime. - All public types implement `Debug` because diagnostic output and test assertions depend on it; for types holding sensitive data, provide a custom implementation that redacts the secret -> [!TIP] -> Prefer `#[derive]` for standard trait implementations. A manual implementation is warranted only when the -> derived behaviour would be incorrect or when redaction is needed. -
Example code: @@ -140,14 +119,15 @@ impl core::fmt::Debug for SecretKey {
-### Error Handling +## Error Handling > [!IMPORTANT] > Never call `.unwrap()` or `.expect()` on `Result` or `Option` in library code. A panic converts a > recoverable failure into process-level failure; depending on the panic strategy, the code may unwind or -> abort, but either outcome is unacceptable for routine error handling. Propagate errors with `?` or handle -> them explicitly with `match`. Both `clippy::unwrap_used` and `clippy::expect_used` are denied at the -> workspace level. +> abort, but either outcome is unacceptable for routine error handling. +> +> Propagate errors with `?` or handle them explicitly with `match`. Both `clippy::unwrap_used` and +> `clippy::expect_used` are denied at the workspace level. - Define domain-specific error enums with a manual `Display` implementation; gate `std::error::Error` behind the `std` feature so the error type remains usable in `no_std` contexts @@ -227,7 +207,7 @@ fn verify_header(raw: &[u8]) -> BlockHeader { -### Ownership and Borrowing +## Ownership and Borrowing Rust's ownership model eliminates data races and use-after-free at compile time. Working with it, rather than around it, produces code that is both safe and efficient. @@ -259,7 +239,7 @@ fn hash_payload(data: &Vec) -> Hash256 { -### Conversions +## Conversions Consistent conversion names tell the reader the cost and ownership semantics of an operation at a glance. @@ -312,7 +292,7 @@ impl BlockHash { -### Traits and Implementations +## Traits and Implementations - Derive standard traits eagerly; the orphan rule prevents downstream crates from adding them, so we provide everything applicable up front @@ -352,7 +332,7 @@ mod private { -### Generics +## Generics - Use `impl Trait` in argument position for simple, single-use bounds; use named type parameters when the same bound appears in multiple arguments or the return type @@ -375,7 +355,7 @@ fn compute_hash(data: impl AsRef<[u8]>) -> Hash256 { -### Iterators +## Iterators Iterator chains express data transformations declaratively. The compiler often optimises them into tight loops with no intermediate allocations. @@ -409,18 +389,18 @@ fn spendable_outputs( -### Code Comments +## Code Comments Comments explain intent and context that the code alone cannot convey. Restating what the code does adds noise and drifts out of sync with the implementation. -#### Inline Comments +### Inline Comments - Line comments (`//`) must not exceed 80 characters wide and 3 lines tall - An extremely complex algorithm may use two short paragraphs separated by a blank comment line - Focus on _why_ a decision was made, not _what_ the code does -#### Rustdoc Comments +### Rustdoc Comments - Documentation comments (`///`) must not exceed 80 characters wide - The summary is at most 3 lines; do not restate the function name or signature in prose because the reader @@ -435,21 +415,18 @@ noise and drifts out of sync with the implementation. Example code: ```rust -/// Decode a compact-encoded block header from raw bytes, -/// verifying the proof-of-work target against the declared -/// difficulty. +/// Decode a compact-encoded block header from raw bytes, verifying the +/// proof-of-work target against the declared difficulty. /// /// # Errors /// -/// Returns `Eof` when the slice holds fewer than 80 bytes, -/// or `BadTarget` when the header fails the proof-of-work -/// threshold. +/// Returns `Eof` when the slice holds fewer than 80 bytes, or `BadTarget` when +/// the header fails the proof-of-work threshold. fn decode_header( raw: &[u8], ) -> Result { - // We validate length before field access to prevent - // out-of-bounds reads when the slice comes from - // malformed or adversarial input. + // We validate length before field access to prevent out-of-bounds reads when + // the slice comes from malformed or adversarial input. if raw.len() < 80 { return Err(DecodeError::Eof { needed: 80, @@ -463,12 +440,9 @@ fn decode_header( ```rust // Bad: restates the signature, wall of text. -/// This function is called decode_header. -/// It takes a byte slice called raw and -/// returns a Result containing either a -/// Header or a DecodeError. The raw -/// parameter is the bytes to decode. If -/// decoding succeeds it returns Ok with +/// This function is called decode_header. It takes a byte slice called raw and +/// returns a Result containing either a Header or a DecodeError. The raw +/// parameter is the bytes to decode. If decoding succeeds it returns Ok with /// the header inside. fn decode_header( raw: &[u8], @@ -503,8 +477,7 @@ fn decode_header( ```rust use core::fmt; -/// 20-byte key hash that can only be constructed from a -/// validating constructor. +/// 20-byte key hash that can only be constructed from a validating constructor. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct KeyId([u8; 20]); @@ -537,6 +510,10 @@ impl KeyId { ### Security +> [!NOTE] +> Audit dependencies regularly. A single compromised or unmaintained transitive dependency can undermine all +> other precautions in the codebase. + - **Never log secrets.** Private keys, key shares, and seed material must never appear in log output, debug strings, or error messages - **Implement `Debug` to redact sensitive fields.** A custom `Debug` that prints a placeholder prevents @@ -549,10 +526,6 @@ impl KeyId { - **Prefer explicit failure over silent defaults.** A default value for a missing secret silently degrades to an insecure state; failing explicitly is always safer than falling back to a placeholder -> [!NOTE] -> Audit dependencies regularly. A single compromised or unmaintained transitive dependency can undermine all -> other precautions in the codebase. -
Example code: From 0c7a789f24d3171e2f3934f867ba08b0cc7eb56c Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Tue, 1 Sep 2026 19:04:38 +0530 Subject: [PATCH 16/18] sdk%doc(zen): add env setup guide, use as base for `Getting Started` --- contrib/README.md | 124 ++++++++++++++++++++++++++++++++++++ docs/dev/getting_started.md | 28 ++++++++ docs/zensical.toml | 1 + 3 files changed, 153 insertions(+) create mode 100644 contrib/README.md create mode 100644 docs/dev/getting_started.md diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 00000000..03c77190 --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,124 @@ + +`base-sdk` uses a family of tools and scripts to maintain correctness and code quality. These scripts are written in +Python 3.x and thus, assume a host capable of running Python. For guidance on installing Python on your host, visit +https://www.python.org/downloads/ + + + +## Preparing the virtual environment + +To avoid conflicting with your existing environment or with Python-based native packages managed by your host, it is +recommended to create a fresh virtual environment. + +> [!NOTE] +> This guide presumes [`uv`](https://github.com/astral-sh/uv) has already been installed on your host. Please refer to +> your program of choice's documentation if using a different manager. + +```bash +# Create a new venv +uv venv .venv + +# Enter venv +source .venv/bin/activate + +# Install dependencies +uv pip install -e ".[dev]" +``` + +> [!WARNING] +> The minimum supported version is Python 3.11, support for prior versions are not expected. If you are running a more +> recent version of Python and are experiencing problems, please +> [file an issue](https://github.com/dashpay/base-sdk/issues/new). + +## Installing dependencies + +[`pyproject.toml`](../pyproject.toml) supplies most but not all dependencies needed to run the lint suite, the following +packages need to be additionally sourced. + +* [Git](https://git-scm.com/install/) +* [CodeQL 2.24 or higher](https://github.com/github/codeql-cli-binaries/releases) (Rust support was added in 2.23.3, + [source](https://github.blog/changelog/2025-10-23-codeql-2-23-3-adds-a-new-rust-query-rust-support-and-easier-c-c-scanning/)) +* [Node.js 24 or higher](https://nodejs.org/en/download) (current LTS, + [source](https://nodejs.org/en/blog/release/v24.11.0)) + +### macOS (with [Homebrew](https://brew.sh/)) + +> [!NOTE] +> Versioned formulae like `node@24` are considered "keg-only", which may require additional steps in order to be +> discoverable in `PATH`, see guidance from Homebrew +> ([source](https://docs.brew.sh/FAQ#what-does-keg-only-mean)). + +```bash +brew install codeql git node@24 +``` + +### Linux/WSL + +See manual installation steps for CodeQL from GitHub +([source](https://docs.github.com/en/code-security/how-tos/find-and-fix-code-vulnerabilities/scan-from-the-command-line/set-up-codeql-cli)), +you may need to update your shell to add your installation path to `PATH` so that `codeql` can be discovered by the +lint script. + +Neither CodeQL nor taplo are available in official Debian or Fedora repositories and must be sourced per vendor +guidance. + +#### Installing `taplo` + +> [!WARNING] +> `.[dev]` doesn't provide `taplo`, needed to run `lint_cargo` on Arm64 Linux. This is due to a release limitation at +> PyPi ([source](https://pypi.org/project/taplo/0.9.3/#files)). The following guidance is not necessary on AMD64 Linux +> or macOS. + +An alternative to procuring releases from the maintainers ([source](https://github.com/tamasfe/taplo/releases)) is to +install it as a Rust binary crate. + +```bash +cargo install taplo-cli +``` + +#### Debian + +```bash +# Required because Debian trixie ships Node 20.x, deprecated in April 2026 +curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash - +sudo apt install git nodejs -y +``` + +#### Fedora + +```bash +sudo dnf install -y git nodejs24 +``` + +## Running linters + +All linters available in [`contrib/lint/`](../contrib/lint) are listed below. The first verb is implied if no verb is +specified at runtime. Verbs may accept arguments of their own, for more information, run an individual lint script with +`--help`. To run all scripts, use [`lint_all.py`](./lint_all.py). + +| Name | Purpose | Verbs | Depends on | +| ---- | ------- | ---------- | ---------- | +| [`lint_cargo.py`](./lint/lint_cargo.py) | Enforce MSRV across Rust build dependency graph, check/format TOML files against [`.taplo.toml`](../.taplo.toml) | `check` , `apply`, `apply-all` | (MSRV enforcement) `cargo` (TOML formatting) `taplo` | +| [`lint_codeql.py`](./lint/lint_codeql.py) | Query Rust sources against [`contrib/codeql/*.ql`](./codeql) | `run` | `codeql`, `rustc` | +| [`lint_javascript.py`](./lint/lint_javascript.py) | Lint Javascript sources against [`eslint.config.mjs`](js/eslint.config.mjs) | *None* | `npx` (part of Node.js), `eslint` (auto-retrieved by script) | +| [`lint_markdown.py`](./lint/lint_markdown.py) | Lint Markdown docs | *None* | `pymarkdownlnt` | +| [`lint_python.py`](./lint/lint_python.py) | Lint Python sources against `[tool.ruff]` options in [`pyproject.toml`](../pyproject.toml) | *None* | `ruff` | +| [`lint_rust.py`](./lint/lint_rust.py) | Lint Rust sources against [`rustfmt.toml`](../rustfmt.toml) | *None* | `cargo`, `rustfmt` | +| [`lint_semgrep.py`](./lint/lint_semgrep.py) | Lint Rust sources against [`contrib/semgrep/*.yml`](./semgrep) | *None* | `semgrep` | +| [`lint_unconv.py`](./lint/lint_unconv.py) | Lint commit names in ranges specified against [`unconv.toml`](../unconv.toml) | *None* | `git` | + +### Verifying bisectability + +As a general rule of thumb, each commit must individually compile and pass linters. To help with this, we have a helper +script, [`git_filter.py`](./git_filter.py) that creates a temporary worktree and executes supplied commands for each +commit in a specified range so the worktree isn't blocked by the validation run. + +```bash +# Replace 'branch_name' with the name of your branch +./contrib/git_filter.py --fast-fail develop branch_name -- bash -c 'cargo clippy --all-targets --no-default-features -- -D warnings && +cargo clippy --all-targets --features full -- -D warnings && +cargo test --all-targets --features full && +./contrib/lint_all.py' +``` + + diff --git a/docs/dev/getting_started.md b/docs/dev/getting_started.md new file mode 100644 index 00000000..9a9406a2 --- /dev/null +++ b/docs/dev/getting_started.md @@ -0,0 +1,28 @@ +# Getting Started + +## Installing Rust + +> [!WARNING] +> Some platforms offer Rust through their package manager. Depending on the maintainer policy, this version may be +> well out of date relative to `stable`. `base-sdk` is only validated against its specified minimum supported Rust +> version (MSRV), its pinned `nightly` and `stable`. +> +> Anything outside that set is untested and may result in unexpected behaviour. + +It is recommended to use [`rustup`](https://rustup.rs/) to manage your Rust build environment. This guide assumes +that platform-specific instructions to install `rustup` have been followed and your `$PATH` variable has been +refreshed. Running it should print something like the following. + +```console +$ rustup --version +rustup 1.29.0 (28d1352db 2026-03-05) +info: This is the version for the rustup toolchain manager, not the rustc compiler. +info: the currently active `rustc` version is `rustc 1.95.0-nightly (905b92696 2026-01-31)` +``` + +By default, `rustup` will read [`rust-toolchain.toml`](../../rust-toolchain.toml) and download the necessary +components at the supported versions without further intervention. Should you want to use a different +version, please consult the vendor documentation for +`RUSTUP_TOOLCHAIN` ([source](https://rust-lang.github.io/rustup/environment-variables.html)). + +--8<-- "contrib/README.md:setup" diff --git a/docs/zensical.toml b/docs/zensical.toml index d3fd6019..a7db52cc 100644 --- a/docs/zensical.toml +++ b/docs/zensical.toml @@ -17,6 +17,7 @@ extra_css = ["style.css"] nav = [ { "Home" = "README.md" }, { "Contributing" = [ + { "Getting Started" = "dev/getting_started.md" }, { "Documentation" = "dev/about_docs.md" }, { "Style Guide (Rust)" = "dev/guide_rust.md" }, ] }, From a5b0a3e774b16584d70bd724accff1cb34df6b56 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Tue, 1 Sep 2026 20:55:53 +0530 Subject: [PATCH 17/18] sdk%doc(zen): drop the samples index for sparseness --- docs/samples/README.md | 12 ------------ docs/zensical.toml | 2 -- 2 files changed, 14 deletions(-) delete mode 100644 docs/samples/README.md diff --git a/docs/samples/README.md b/docs/samples/README.md deleted file mode 100644 index 32100729..00000000 --- a/docs/samples/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Samples - -To demonstrate the SDK's capabilities, the Base SDK comes with sample code demonstrating various protocol operations. -These samples are interactive and can be run directly within the guide. - -Native samples are `std`-dependent while web-based samples are built with [`wasm-pack`](https://wasm-bindgen.github.io/wasm-pack/installer) -targeting the `web` platform. - -| Name | Description | Web-based | -| ---- | ----------- | --------- | -| [Genesis Solver](solver/README.md) | Verify or mint a genesis block (uses `dash-pow`). | Yes | -| [Object Parser](parser/README.md) | Parse hex-encoded blocks or transactions into a navigable tree (uses `dash-primitives`). | Yes | diff --git a/docs/zensical.toml b/docs/zensical.toml index a7db52cc..34f1a990 100644 --- a/docs/zensical.toml +++ b/docs/zensical.toml @@ -22,7 +22,6 @@ nav = [ { "Style Guide (Rust)" = "dev/guide_rust.md" }, ] }, { "Samples" = [ - { "Samples" = "samples/README.md" }, { "Genesis Solver" = "samples/solver/README.md" }, { "Object Parser" = "samples/parser/README.md" }, ] }, @@ -39,7 +38,6 @@ features = [ "content.tabs.link", "navigation.expand", "navigation.footer", - "navigation.indexes", "navigation.instant.prefetch", "navigation.instant", "navigation.sections", From e57239b01ecde6c34a54e63db4860abfacc5f690 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:46:48 +0530 Subject: [PATCH 18/18] sdk%doc(zen): update documentation page to be more informative --- contrib/README.md | 2 +- docs/dev/about_docs.md | 96 +++++++++++++++++++++++++++++++++++------- 2 files changed, 82 insertions(+), 16 deletions(-) diff --git a/contrib/README.md b/contrib/README.md index 03c77190..8da1b40a 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -101,7 +101,7 @@ specified at runtime. Verbs may accept arguments of their own, for more informat | [`lint_cargo.py`](./lint/lint_cargo.py) | Enforce MSRV across Rust build dependency graph, check/format TOML files against [`.taplo.toml`](../.taplo.toml) | `check` , `apply`, `apply-all` | (MSRV enforcement) `cargo` (TOML formatting) `taplo` | | [`lint_codeql.py`](./lint/lint_codeql.py) | Query Rust sources against [`contrib/codeql/*.ql`](./codeql) | `run` | `codeql`, `rustc` | | [`lint_javascript.py`](./lint/lint_javascript.py) | Lint Javascript sources against [`eslint.config.mjs`](js/eslint.config.mjs) | *None* | `npx` (part of Node.js), `eslint` (auto-retrieved by script) | -| [`lint_markdown.py`](./lint/lint_markdown.py) | Lint Markdown docs | *None* | `pymarkdownlnt` | +| [`lint_markdown.py`](./lint/lint_markdown.py) | Lint Markdown [documentation](../docs/dev/about_docs.md) | *None* | `pymarkdownlnt` | | [`lint_python.py`](./lint/lint_python.py) | Lint Python sources against `[tool.ruff]` options in [`pyproject.toml`](../pyproject.toml) | *None* | `ruff` | | [`lint_rust.py`](./lint/lint_rust.py) | Lint Rust sources against [`rustfmt.toml`](../rustfmt.toml) | *None* | `cargo`, `rustfmt` | | [`lint_semgrep.py`](./lint/lint_semgrep.py) | Lint Rust sources against [`contrib/semgrep/*.yml`](./semgrep) | *None* | `semgrep` | diff --git a/docs/dev/about_docs.md b/docs/dev/about_docs.md index 5e3c2d38..28f4e541 100644 --- a/docs/dev/about_docs.md +++ b/docs/dev/about_docs.md @@ -1,26 +1,92 @@ -# Building Docs +# Documentation -The guide is generated using [Zensical](https://pypi.org/project/zensical/) (a fork of -[MkDocs](https://pypi.org/project/mkdocs/)), configured using -[`zensical.toml`](../zensical.toml). Every source file it needs -lives in [`docs/`](..), the rendered site is written to -`docs/.site`. +This guide is generated using [Zensical](https://pypi.org/project/zensical/) (a fork of +[MkDocs](https://pypi.org/project/mkdocs/)), with additional [pre](#preprocessing)- and +[post](#postprocessing)-processing to make the source material render adequately on the forge provider, GitHub. + +## Installing dependencies + +> [!NOTE] +> If you haven't set up your development environment, check out the [startup guide](./getting_started.md) first. + +The documentation comes bundled with web-ready demos, which are powered by WebAssembly. Preparing them for +distribution relies on [`wasm-pack`](https://github.com/wasm-bindgen/wasm-pack), which is installed as a binary crate. + +```bash +cargo install wasm-pack +``` -Most dependencies can be installed using `python -m pip install -e '.[dev]'`. +## Building -* Zensical (included in `[dev]`) -* PyMarkdown (included in `[dev]`) -* rjsmin (included in `[dev]`) -* [wasm-pack](https://github.com/wasm-bindgen/wasm-pack) +> [!WARNING] +> Due to a limitation in Zensical, the target bundle cannot be emitted in the usual `public/` directory. +> This is because a target path must be at or in a child directory relative to [`zensical.toml`](../zensical.toml). +> For more information, see [zensical/backlog#56](https://github.com/zensical/backlog/issues/56). -From the repository root, render the site with +To generate the target bundle, run the following from the repository root. The target bundle will be located at +`docs/.site`. -```sh +```bash python docs/build_docs.py build ``` -or render it and serve it over localhost with +### Preview + +> [!TIP] +> If a preview appears stale even after rebuilding, you may need to clear `docs/.{cache,site}`. + +To preview the site live, run the following from the repository root. -```sh +```bash python docs/build_docs.py preview ``` + +Live reload is unsupported due to additional processing and compilation artifacts. Using `zensical` directly for +previews or builds may result in a broken site. + +## Preprocessing + +This documentation has two audiences, GitHub and Zensical. This created an impasse between two competing Markdown +extensions, [GitHub-flavoured Markdown](https://github.github.com/gfm/) (GFM) and +[Python-Markdown](https://pypi.org/project/Markdown/) syntax. Two measures resolve it. + +* Naming base pages `README.md` instead of `index.md`, so that GitHub renders a directory's associated base page instead + of returning empty. +* Processing Markdown files (see [`preprocess.py`](../preprocess.py)) through a Python-Markdown extension before it is + rendered by Zensical as a webpage. + +### Admonitions + +The Python ecosystem settled on a syntax for [admonitions](https://zensical.org/docs/authoring/admonitions/) that can +then be extended by Zensical to offer arbitrary icons and accent colors with adequate theming. By contrast, GFM alerts +are relatively rigid but are broadly supported in the Markdown ecosystem (including by WYSIWYG editors like +[Typora](https://typora.io/)). + +To bridge this gap, alerts are mapped to the nearest fitting admonition, `[!IMPORTANT]` becomes `info` and `[!CAUTION]` +becomes `danger`. + +### Link processing + +> [!TIP] +> To cite specific line (ranges), it is advised to link against a commit-pinned version of that file on GitHub (or +> elsewhere reachable on the open web) to ensure that the ranges don't turn stale as the codebase evolves. + +On-disk link targets are written relative to the file defining it, as only files under `docs/` are carried into the +target bundle. To allow links outside `docs/` to resolve to a valid path, links pointing to valid on-disk elements +outside `docs/` resolve to the forge instead. + +> [!WARNING] +> Zensical treats on-disk `.md` links as documentation and will fail to build if they are located outside `docs/`. +> This does not affect non-Markdown files and directories. + +## Postprocessing + +> [!WARNING] +> The following is a workaround. Zensical does not offer a setting to hold a file back from the target bundle. +> Support for `not_in_nav` ([zensical/backlog#63](https://github.com/zensical/backlog/issues/63)) as well as +> `exclude_docs` and `draft_docs` ([zensical/backlog#65](https://github.com/zensical/backlog/issues/65)) is pending. + +Zensical copies the whole of `docs/` into the target bundle, so sources, manifests, scripts and other non-publishable +materials are included in the target bundle and may end up exposed on the open web. To avoid this, +[`.zenignore`](../.zenignore) lists globs for elements to be excluded from the target bundle. **While `.zenignore` has +similar syntax to `.gitignore`, exclusions (`!`) are not supported and will result in a hard error.**