diff --git a/.gitignore b/.gitignore index 3096237c..d80e42e8 100644 --- a/.gitignore +++ b/.gitignore @@ -29,5 +29,6 @@ grimoires/ # Superpowers SDD scratch workspace (briefs, reports, review packages, ledger) .superpowers/ -# Superpowers implementation plans — kept local, not committed (specs are committed) -docs/superpowers/plans/ +# Superpowers implementation plans are local scratch artifacts, never commit them. +/docs/superpowers/plans/ +/docs/superpowers/plans/**/*.md diff --git a/AGENTS.md b/AGENTS.md index 5232632b..1cb03a83 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -176,7 +176,9 @@ Benchmarks are opt-in: `./gradlew benchmarkCompare -Pbenchmark=true`. Runner tas `ext.benchmarkRunner = true`; the root aggregator auto-discovers them. New runners must use the shared corpus/schema and write `benchmarks/results/-.json`. Do not add benchmark execution to normal `build` or `test`. The CLI benchmark requires a -binary built with `-Pbenchmark=true`; `DW_BENCH_BIN` may select a prebuilt one. +normal `dw` binary. `-Pbenchmark=true` gates Gradle benchmark task execution; it does not +make the artifact benchmark-capable. `DW_BENCH_BIN` selects an existing ordinary `dw` +binary. CI builds Ubuntu and Windows with GraalVM 24. Native CLI regression suites and Node TCK run only on `master`. Run the smallest relevant suite, then the nearest module test; use diff --git a/benchmarks/README.md b/benchmarks/README.md index 3b1dfe3d..ae59e4d1 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -13,12 +13,8 @@ Language-agnostic benchmark harness for the DataWeave native-lib wrappers. (Scala/Gradle subproject `:benchmarks-engine`, depends on `org.mule.weave:runtime` at the same `weaveVersion` the native image is built from). `runners/python/` is the Python runner (stdlib scripts under `native-lib`, wrapping the same staged `dwlib` as Node). - `runners/cli/` is the CLI runner: a Node parent that spawns the `dw` native - binary (built with `-Pbenchmark=true`, which compiles in an in-binary - benchmark harness gated by `BenchmarkMode.ENABLED` and dispatched via the - `DW_BENCH` env var — the shipped `dw` contains none of it). It emits - `cold-start`, `first-run`, and `warm`; it does **not** emit `streaming` - (the `dw run` path has no chunked-input FFI like the library's). + `runners/cli/` is the CLI runner: a Node parent that spawns normal `dw run` + commands and emits only end-to-end `first-run` measurements. - `report/report.mjs` — joins result files against the manifest and prints a comparison table. - `results/` — gitignored per-run output. @@ -27,14 +23,20 @@ Language-agnostic benchmark harness for the DataWeave native-lib wrappers. `cold-start` and `first-run` (fresh process per sample), `warm` (in-process steady state), `streaming` (MB/s). Each case declares which apply via `metrics[]`. -**Cold-start is measured by the parent, not the child** — every runner spawns a fresh -child that prints a `READY` marker the instant its runtime is initialized, and the parent -records wall-clock from just-before-spawn to that marker. So cold-start includes process -launch + library/class load + runtime init on all three runners, which is what makes the -native-image-vs-JVM comparison meaningful (the native image has no JVM to boot; the JVM's -cold cost *is* launch + classload). Adding a runner requires the same protocol: print -`READY` (flushed) after init, then a JSON line with the in-process `firstRunMs`. Note only -the first sample sees a truly cold OS page cache; the reported median is warm-cache init. +For the CLI runner, `first-run` is end-to-end `dw run` command latency. Other +runners' `first-run` is in-process compile-and-execute latency. The CLI emits +no `cold-start`, `warm`, or `streaming` rows, so its table deltas remain visible +but qualify a different measurement boundary. + +**Cold-start is measured by the parent, not the child** for the Node, Python, and engine +runners. Their fresh child prints a `READY` marker the instant its runtime is initialized, +and the parent records wall-clock from just-before-spawn to that marker. Cold-start therefore +includes process launch + library/class load + runtime init, which makes the native-image-vs-JVM +comparison meaningful (the native image has no JVM to boot; the JVM's cold cost *is* launch + +classload). These in-process runners use the `READY` (flushed) plus JSON `firstRunMs` protocol. +The CLI does not use that protocol: it measures each normal `dw run` process from spawn to +successful exit. Only the first sample sees a truly cold OS page cache; the reported median is +warm-cache init. ## Prerequisites @@ -45,9 +47,8 @@ and `JAVA_HOME` set to it (see the root README / `CLAUDE.md`). The pinned build `graalvmVersion` in `gradle.properties`. The **engine runner alone** drives the JVM `DataWeaveScriptingEngine` and runs on any JDK — no native image required. -The **CLI runner** requires the bench-enabled binary -(`./gradlew native-cli:nativeCompile -Pbenchmark=true`); set `DW_BENCH_BIN` to -point at a prebuilt one. Like the library runners it needs the GraalVM toolchain. +`DW_BENCH_BIN` points to an ordinary prebuilt `dw` binary; the CLI runner does +not build it when the override is supplied. ## Running @@ -55,9 +56,37 @@ The one-shot cross-runner comparison — runs **every** registered runner and pr ./gradlew benchmarkCompare -Pbenchmark=true # all runners + comparison report +### Running against pre-built wrapper artifacts + +The Node and Python runners can benchmark pre-built wrapper artifacts via env vars, skipping +their corresponding local wrapper build or staging task: + +- **`DW_BENCH_NODE_PACKAGE`** — absolute path to an extracted `@dataweave/native` package + directory (must contain `dist/index.js`). Example: + + DW_BENCH_NODE_PACKAGE=/tmp/artifacts/node/package \ + ./gradlew native-lib:benchmarkNode -Pbenchmark=true + +- **`DW_BENCH_PY_SITE`** — absolute path to a site-packages-style directory containing + `dataweave/__init__.py`. Populate with `pip install --target `. Example: + + pip install --target /tmp/artifacts/py dataweave-0.0.1-py3-none-any.whl + DW_BENCH_PY_SITE=/tmp/artifacts/py \ + ./gradlew native-lib:benchmarkPython -Pbenchmark=true + +If the env var is set but the target is invalid, the runner fails immediately rather than +falling back to the source tree. For a cross-runner comparison with both wrapper overrides: + + DW_BENCH_NODE_PACKAGE=/tmp/artifacts/node/package \ + DW_BENCH_PY_SITE=/tmp/artifacts/py \ + ./gradlew benchmarkCompare -Pbenchmark=true + +Use `DW_BENCH_BIN` to point the CLI runner at an ordinary prebuilt `dw` binary; +when it is set, `benchmarkCli` does not run a local `nativeCompile`. + Single-runner options: - ./gradlew native-lib:benchmark -Pbenchmark=true # Node only: build wrapper, run, report + ./gradlew native-lib:benchmarkNode -Pbenchmark=true # Node only: writes results/node-.json ./gradlew benchmarks-engine:benchmarkEngine -Pbenchmark=true # engine (JVM) only: writes results/engine-.json ./gradlew native-lib:benchmarkPython -Pbenchmark=true # Python only: writes results/python-.json ./gradlew native-cli:benchmarkCli -Pbenchmark=true # CLI only: writes results/cli-.json @@ -68,6 +97,11 @@ Or directly, once the wrapper is built (`./gradlew native-lib:buildNodePackage`) node runners/node/emit.mjs # writes results/node-.json node report/report.mjs results/*.json # renders the table +`gen-inputs.mjs` reuses an existing `corpus/inputs/generated/records-large.json` so +every runner in a comparison uses the same bytes. `BENCH_LARGE_N` is applied only when +the file is first generated; delete `corpus/inputs/generated/records-large.json` before +running the generator to create a corpus with a different record count. + Results (`results/*.json`) are local-only and gitignored; no history is accumulated (see the design spec). To publish a snapshot, render a self-contained Markdown report with charts: diff --git a/benchmarks/corpus/gen-inputs.mjs b/benchmarks/corpus/gen-inputs.mjs index ddd4d72e..c68fdc99 100644 --- a/benchmarks/corpus/gen-inputs.mjs +++ b/benchmarks/corpus/gen-inputs.mjs @@ -1,6 +1,6 @@ // Deterministically regenerate large inputs. No randomness -> comparable across // machines and runners. Size overridable via BENCH_LARGE_N (default 50000). -import { writeFileSync, mkdirSync } from "node:fs"; +import { existsSync, statSync, writeFileSync, mkdirSync } from "node:fs"; import { join, dirname } from "node:path"; import { fileURLToPath } from "node:url"; @@ -9,10 +9,15 @@ const outDir = join(__dirname, "inputs", "generated"); mkdirSync(outDir, { recursive: true }); const n = Number(process.env.BENCH_LARGE_N ?? 50000); +const path = join(outDir, "records-large.json"); +if (existsSync(path)) { + console.log(`reusing ${statSync(path).size} byte input at ${path}`); + process.exit(0); +} + const records = []; for (let i = 1; i <= n; i++) { records.push({ id: i, name: `item_${i}`, value: i * 3 }); } -const path = join(outDir, "records-large.json"); writeFileSync(path, JSON.stringify(records)); console.log(`wrote ${n} records to ${path}`); diff --git a/benchmarks/lib/env.mjs b/benchmarks/lib/env.mjs index ea5d9c80..b9ec66e2 100644 --- a/benchmarks/lib/env.mjs +++ b/benchmarks/lib/env.mjs @@ -24,12 +24,15 @@ function readCommit() { } } -// Best-effort identity of the staged dwlib: first 8 hex of a sha256 over +// Best-effort identity of the selected dwlib: first 8 hex of a sha256 over // (size + first 64KB). Cheap, stable, and enough to detect a lib swap. -function readDwlibBuildId() { - const base = join(REPO_ROOT, "native-lib", "node", "native"); - for (const ext of [".dylib", ".so", ".dll"]) { - const p = join(base, `dwlib${ext}`); +function readDwlibBuildId(dwlibPath) { + const paths = dwlibPath && existsSync(dwlibPath) + ? [dwlibPath] + : [".dylib", ".so", ".dll"].map((ext) => { + return join(REPO_ROOT, "native-lib", "node", "native", `dwlib${ext}`); + }); + for (const p of paths) { if (existsSync(p)) { const buf = readFileSync(p).subarray(0, 65536); const size = statSync(p).size; @@ -40,9 +43,9 @@ function readDwlibBuildId() { } /** - * @param {{runner:string, runtimeVersion:string}} opts + * @param {{runner:string, runtimeVersion:string, dwlibPath?:string}} opts */ -export function gatherEnv({ runner, runtimeVersion }) { +export function gatherEnv({ runner, runtimeVersion, dwlibPath }) { const cpus = os.cpus(); return { runner, @@ -51,6 +54,6 @@ export function gatherEnv({ runner, runtimeVersion }) { runtimeVersion, weaveVersion: readWeaveVersion(), commit: readCommit(), - dwlibBuildId: readDwlibBuildId(), + dwlibBuildId: readDwlibBuildId(dwlibPath), }; } diff --git a/benchmarks/lib/env.test.mjs b/benchmarks/lib/env.test.mjs index f37f446c..77e36d69 100644 --- a/benchmarks/lib/env.test.mjs +++ b/benchmarks/lib/env.test.mjs @@ -1,7 +1,25 @@ import { test } from "node:test"; import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; import { gatherEnv } from "./env.mjs"; +const tempDirs = []; + +function makeTempDir() { + const dir = mkdtempSync(join(tmpdir(), "dw-bench-env-test-")); + tempDirs.push(dir); + return dir; +} + +test.after(() => { + for (const dir of tempDirs) { + rmSync(dir, { recursive: true, force: true }); + } +}); + test("gatherEnv returns all required fields", () => { const env = gatherEnv({ runner: "node-wrapper", runtimeVersion: "node vX" }); for (const key of ["os", "cpu", "runtimeVersion", "weaveVersion", "commit", "dwlibBuildId"]) { @@ -14,3 +32,22 @@ test("gatherEnv reads the pinned weaveVersion from gradle.properties", () => { // gradle.properties pins e.g. 2.12.0-YYYYMMDD; assert it looks like a weave version. assert.match(env.weaveVersion, /^\d+\.\d+\.\d+/); }); + +test("gatherEnv attributes an explicitly selected native library", () => { + const libraryPath = join(makeTempDir(), "dwlib.dylib"); + const libraryBytes = Buffer.from("external native library fixture"); + writeFileSync(libraryPath, libraryBytes); + const expectedBuildId = "dwlib-" + createHash("sha256") + .update(String(libraryBytes.length)) + .update(libraryBytes.subarray(0, 65536)) + .digest("hex") + .slice(0, 8); + + const env = gatherEnv({ + runner: "node-wrapper", + runtimeVersion: "node vX", + dwlibPath: libraryPath, + }); + + assert.equal(env.dwlibBuildId, expectedBuildId); +}); diff --git a/benchmarks/report/report.mjs b/benchmarks/report/report.mjs index e07486ab..228c53cb 100644 --- a/benchmarks/report/report.mjs +++ b/benchmarks/report/report.mjs @@ -150,6 +150,12 @@ export function renderMermaidCharts(table) { return blocks.join("\n\n"); } +export function renderMetricNotes(results) { + if (!results.some((result) => result.runner === "cli")) return ""; + return "CLI `first-run` is end-to-end `dw run` command latency. " + + "Other runners' `first-run` is in-process compile-and-execute latency."; +} + /** * A self-contained Markdown report: provenance (commit + date), the numeric * table, then a Mermaid bar chart per (case, metric) — one bar per runner. @@ -168,6 +174,8 @@ export function renderMarkdown(table, results, { baselineRunner, stamp }) { "> Indicative only — timings are from a single run on one machine, not a dedicated bench box.", "" ); + const metricNotes = renderMetricNotes(results); + if (metricNotes) out.push(metricNotes, ""); out.push("## Table", ""); out.push("| " + table.header.join(" | ") + " |"); @@ -214,6 +222,11 @@ export function main(argv) { console.log(`⚠️ WEAVE VERSION SKEW: comparing across ${skew.join(" vs ")} — deltas are not clean.`); console.log(""); } + const metricNotes = renderMetricNotes(results); + if (metricNotes) { + console.log(metricNotes); + console.log(""); + } const table = buildTable(manifest, results, baselineRunner); const { header, rows, otherRunners } = table; diff --git a/benchmarks/report/report.test.mjs b/benchmarks/report/report.test.mjs index e7adf34d..acb875d7 100644 --- a/benchmarks/report/report.test.mjs +++ b/benchmarks/report/report.test.mjs @@ -11,6 +11,7 @@ import { buildTable, dedupeLatestByRunner, renderMermaidCharts, + renderMetricNotes, renderMarkdown, } from "./report.mjs"; @@ -82,6 +83,33 @@ test("renderMarkdown emits no streaming non-comparable footnote", () => { }); assert.ok(md.includes("| map-scale | streaming | MB/s |"), "streaming row is present"); assert.ok(!md.includes("not like-for-like across runners"), "footnote removed"); + assert.equal(renderMetricNotes(results), "", "metric note is omitted without CLI results"); +}); + +test("report rendering labels CLI first-run as end-to-end and other runners as in-process", () => { + const manifest = loadManifest(CORPUS); + const engine = load("engine-b.json"); + const cli = { + ...engine, + runner: "cli", + cases: engine.cases + .filter((result) => result.metric === "first-run") + .map((result) => ({ ...result })), + }; + const results = [engine, cli]; + const table = buildTable(manifest, results, "engine"); + const md = renderMarkdown(table, results, { + baselineRunner: "engine", + stamp: { commit: "abc1234", date: "2026-08-10T14:33:03Z" }, + }); + + assert.ok(md.includes("CLI `first-run` is end-to-end `dw run` command latency.")); + assert.ok(md.includes("Other runners' `first-run` is in-process compile-and-execute latency.")); + assert.equal( + renderMetricNotes(results), + "CLI `first-run` is end-to-end `dw run` command latency. " + + "Other runners' `first-run` is in-process compile-and-execute latency." + ); }); test("renderMermaidCharts emits one chart per (case, metric) with a bar per runner", () => { diff --git a/benchmarks/runners/cli/coldstart.mjs b/benchmarks/runners/cli/coldstart.mjs deleted file mode 100644 index caca5d6e..00000000 --- a/benchmarks/runners/cli/coldstart.mjs +++ /dev/null @@ -1,100 +0,0 @@ -import { spawn } from "node:child_process"; -import { join } from "node:path"; -import { casesForMetric } from "../../lib/manifest.mjs"; -import { computeStats } from "../../lib/stats.mjs"; -import { locateBinary } from "./locate.mjs"; - -/** Build `--input=name=file\tmime\tcharset` args for a case (absolute paths). */ -function inputArgs(manifest, c) { - const args = []; - for (const [name, inp] of Object.entries(c.inputs ?? {})) { - const file = join(manifest.corpusDir, inp.file); - const charset = inp.charset ?? "utf-8"; - args.push(`--input=${name}=${file}\t${inp.mimeType}\t${charset}`); - } - return args; -} - -/** - * Spawn one fresh dw process in coldfirst mode. Cold-start = wall-clock from just - * before spawn to the child's "READY" marker (process launch + native image load + - * NativeRuntime init). first-run is timed in-process by the child. Rejects on a - * non-zero exit or a missing READY/JSON line so a failed sample never records a - * bogus timing. - */ -function sampleOnce(bin, manifest, c) { - const scriptPath = join(manifest.corpusDir, c.script); - const args = ["--bench-mode=coldfirst", `--script=${scriptPath}`, ...inputArgs(manifest, c)]; - return new Promise((resolve, reject) => { - const t0 = process.hrtime.bigint(); - const child = spawn(bin, args, { - stdio: ["ignore", "pipe", "pipe"], - env: { ...process.env, DW_BENCH: "1" }, - }); - let coldStartMs; - let stdout = ""; - let stderr = ""; - child.stdout.setEncoding("utf-8"); - child.stdout.on("data", (chunk) => { - stdout += chunk; - if (coldStartMs === undefined && stdout.includes("READY\n")) { - coldStartMs = Number(process.hrtime.bigint() - t0) / 1e6; - } - }); - child.stderr.setEncoding("utf-8"); - child.stderr.on("data", (chunk) => (stderr += chunk)); - child.on("error", reject); - child.on("close", (code) => { - if (code !== 0) { - reject(new Error(`cli coldfirst failed for '${c.id}' (exit ${code})\n${stderr}`)); - return; - } - if (coldStartMs === undefined) { - reject(new Error(`cli coldfirst for '${c.id}' never printed READY\n${stderr}`)); - return; - } - const jsonLine = stdout.split("\n").filter((l) => l && l !== "READY").pop(); - if (!jsonLine) { - reject(new Error(`cli coldfirst for '${c.id}' printed no result line\n${stderr}`)); - return; - } - let firstRunMs; - try { - ({ firstRunMs } = JSON.parse(jsonLine)); - } catch (error) { - reject(new Error(`cli coldfirst for '${c.id}' printed invalid JSON: ${jsonLine}`, { cause: error })); - return; - } - resolve({ coldStartMs, firstRunMs }); - }); - }); -} - -/** @returns {Promise>} */ -export async function runColdStartAndFirstRun(manifest, { samplesOverride } = {}) { - const bin = locateBinary(); - const rows = []; - const ids = new Set([ - ...casesForMetric(manifest, "cold-start").map((c) => c.id), - ...casesForMetric(manifest, "first-run").map((c) => c.id), - ]); - - for (const id of ids) { - const c = manifest.cases.find((x) => x.id === id); - const n = samplesOverride ?? c.iterations?.samples ?? 20; - const colds = []; - const firsts = []; - for (let i = 0; i < n; i++) { - const { coldStartMs, firstRunMs } = await sampleOnce(bin, manifest, c); - colds.push(coldStartMs); - firsts.push(firstRunMs); - } - if (c.metrics.includes("cold-start")) { - rows.push({ id, metric: "cold-start", unit: "ms", stats: computeStats(colds), iterations: n }); - } - if (c.metrics.includes("first-run")) { - rows.push({ id, metric: "first-run", unit: "ms", stats: computeStats(firsts), iterations: n }); - } - } - return rows; -} diff --git a/benchmarks/runners/cli/emit.mjs b/benchmarks/runners/cli/emit.mjs index 699667d0..8a877620 100644 --- a/benchmarks/runners/cli/emit.mjs +++ b/benchmarks/runners/cli/emit.mjs @@ -5,8 +5,7 @@ import { fileURLToPath, pathToFileURL } from "node:url"; import { loadManifest, validateResultIds } from "../../lib/manifest.mjs"; import { gatherEnv } from "../../lib/env.mjs"; import { locateBinary } from "./locate.mjs"; -import { runColdStartAndFirstRun } from "./coldstart.mjs"; -import { runWarm } from "./warm.mjs"; +import { runFirstRun } from "./first-run.mjs"; const __dirname = dirname(fileURLToPath(import.meta.url)); const CORPUS = join(__dirname, "..", "..", "corpus"); @@ -41,10 +40,7 @@ export async function main() { // The CLI is a native binary, not the staged dwlib — override the lib fingerprint. env.dwlibBuildId = "n/a-cli"; - const coldRows = await runColdStartAndFirstRun(manifest); - const warmRows = await runWarm(manifest); - - const cases = [...coldRows, ...warmRows]; + const cases = await runFirstRun(manifest); validateResultIds(manifest, cases); mkdirSync(RESULTS_DIR, { recursive: true }); diff --git a/benchmarks/runners/cli/emit.test.mjs b/benchmarks/runners/cli/emit.test.mjs index 88d15870..f8537ff5 100644 --- a/benchmarks/runners/cli/emit.test.mjs +++ b/benchmarks/runners/cli/emit.test.mjs @@ -13,7 +13,7 @@ test("buildResult produces a schema-shaped object with runner 'cli'", () => { runner: "cli", os: "x", cpu: "y", runtimeVersion: "dw vX", weaveVersion: "2.12.0-x", commit: "abc", dwlibBuildId: "n/a-cli", }; - const cases = [{ id: "trivial", metric: "cold-start", unit: "ms", stats: { median: 1 }, iterations: 10 }]; + const cases = [{ id: "trivial", metric: "first-run", unit: "ms", stats: { median: 1 }, iterations: 10 }]; const r = buildResult(env, cases); assert.equal(r.schemaVersion, "1.0"); assert.equal(r.runner, "cli"); diff --git a/benchmarks/runners/cli/first-run.mjs b/benchmarks/runners/cli/first-run.mjs new file mode 100644 index 00000000..594ffaac --- /dev/null +++ b/benchmarks/runners/cli/first-run.mjs @@ -0,0 +1,51 @@ +import { spawn } from "node:child_process"; +import { join } from "node:path"; +import { casesForMetric } from "../../lib/manifest.mjs"; +import { computeStats } from "../../lib/stats.mjs"; +import { locateBinary } from "./locate.mjs"; + +function commandArgs(manifest, c) { + const args = ["run"]; + for (const [name, input] of Object.entries(c.inputs ?? {})) { + args.push("-i", `${name}=${join(manifest.corpusDir, input.file)}`); + } + args.push("--file", join(manifest.corpusDir, c.script)); + return args; +} + +export function sampleOnce(bin, args, c, { spawnFn = spawn } = {}) { + return new Promise((resolve, reject) => { + const t0 = process.hrtime.bigint(); + const child = spawnFn(bin, args, { stdio: ["ignore", "ignore", "pipe"] }); + let stderr = ""; + child.stderr.setEncoding("utf-8"); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.on("error", (error) => { + reject(new Error(`cli first-run failed for '${c.id}'\n${stderr}`, { cause: error })); + }); + child.on("close", (code) => { + if (code !== 0) { + reject(new Error(`cli first-run failed for '${c.id}' (exit ${code})\n${stderr}`)); + return; + } + resolve(Number(process.hrtime.bigint() - t0) / 1e6); + }); + }); +} + +/** @returns {Promise>} */ +export async function runFirstRun(manifest, { sample: sampleOverride, binary, samplesOverride } = {}) { + const bin = binary ?? locateBinary(); + const sampleFn = sampleOverride ?? sampleOnce; + const rows = []; + for (const c of casesForMetric(manifest, "first-run")) { + const n = samplesOverride ?? c.iterations?.samples ?? 20; + const samples = []; + const args = commandArgs(manifest, c); + for (let i = 0; i < n; i++) { + samples.push(await sampleFn(bin, args, c)); + } + rows.push({ id: c.id, metric: "first-run", unit: "ms", stats: computeStats(samples), iterations: n }); + } + return rows; +} diff --git a/benchmarks/runners/cli/first-run.test.mjs b/benchmarks/runners/cli/first-run.test.mjs new file mode 100644 index 00000000..c50cdfed --- /dev/null +++ b/benchmarks/runners/cli/first-run.test.mjs @@ -0,0 +1,102 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { PassThrough } from "node:stream"; +import { EventEmitter } from "node:events"; +import { join } from "node:path"; +import { runFirstRun, sampleOnce } from "./first-run.mjs"; + +const corpusDir = "/fake/corpus"; +const script = join(corpusDir, "scripts/object-transform.dwl"); +const inputPath = join(corpusDir, "inputs/person-record.json"); +const manifest = { + corpusDir, + cases: [{ + id: "object-transform", + script: "scripts/object-transform.dwl", + inputs: { + payload: { file: "inputs/person-record.json", mimeType: "application/json" }, + }, + metrics: ["first-run"], + }], +}; + +function childProcess() { + const child = new EventEmitter(); + child.stderr = new PassThrough(); + return child; +} + +test("sampleOnce resolves elapsed time after a successful child close", async () => { + const child = childProcess(); + const elapsed = await sampleOnce("/fake/dw", ["run"], manifest.cases[0], { + spawnFn: () => { + queueMicrotask(() => child.emit("close", 0)); + return child; + }, + }); + + assert.equal(typeof elapsed, "number"); + assert.ok(elapsed >= 0); +}); + +test("sampleOnce rejects a nonzero close with captured stderr", async () => { + const child = childProcess(); + await assert.rejects( + sampleOnce("/fake/dw", ["run"], manifest.cases[0], { + spawnFn: () => { + queueMicrotask(() => { + child.stderr.write("invalid script"); + child.emit("close", 2); + }); + return child; + }, + }), + /cli first-run failed for 'object-transform' \(exit 2\)\ninvalid script/, + ); +}); + +test("sampleOnce rejects a child spawn error", async () => { + const child = childProcess(); + await assert.rejects( + sampleOnce("/fake/dw", ["run"], manifest.cases[0], { + spawnFn: () => { + queueMicrotask(() => child.emit("error", new Error("not executable"))); + return child; + }, + }), + /cli first-run failed for 'object-transform'/, + ); +}); + +test("runFirstRun samples ordinary dw run arguments and aggregates samples", async () => { + const rows = await runFirstRun(manifest, { + sample: async (bin, args) => { + assert.equal(bin, "/fake/dw"); + assert.deepEqual(args, ["run", "-i", `payload=${inputPath}`, "--file", script]); + return 12.5; + }, + binary: "/fake/dw", + samplesOverride: 2, + }); + + assert.deepEqual(rows, [{ + id: "object-transform", + metric: "first-run", + unit: "ms", + stats: { min: 12.5, median: 12.5, p90: 12.5, p99: 12.5, mean: 12.5 }, + iterations: 2, + }]); +}); + +test("runFirstRun rejects instead of returning rows after a sample failure", async () => { + await assert.rejects( + runFirstRun(manifest, { + sample: async () => { + throw new Error("cli first-run failed for 'object-transform'"); + }, + binary: "/fake/dw", + samplesOverride: 2, + }), + /cli first-run failed for 'object-transform'/, + ); +}); diff --git a/benchmarks/runners/cli/locate.mjs b/benchmarks/runners/cli/locate.mjs index d8112077..4bcdb066 100644 --- a/benchmarks/runners/cli/locate.mjs +++ b/benchmarks/runners/cli/locate.mjs @@ -9,17 +9,16 @@ const BIN_NAME = process.platform === "win32" ? "dw.exe" : "dw"; const DEFAULT_BIN = join(REPO_ROOT, "native-cli", "build", "native", "nativeCompile", BIN_NAME); /** - * Resolve the benchmark-enabled `dw` native binary. Honors DW_BENCH_BIN (absolute - * path to a bench-built dw); otherwise the default nativeCompile output. The binary - * must be built with -Pbenchmark=true so BenchmarkHarness is reachable. + * Resolve a `dw` native executable. Honors DW_BENCH_BIN (absolute path to an + * executable); otherwise the default nativeCompile output. */ export function locateBinary() { const candidate = process.env.DW_BENCH_BIN || DEFAULT_BIN; if (!existsSync(candidate)) { throw new Error( - `dw benchmark binary not found at ${candidate}. ` + - `Build it with: ./gradlew native-cli:nativeCompile -Pbenchmark=true ` + - `(or set DW_BENCH_BIN to a bench-enabled dw).` + `dw binary not found at ${candidate}. ` + + `Build it with: ./gradlew native-cli:nativeCompile ` + + `(or set DW_BENCH_BIN to a dw executable).` ); } return candidate; diff --git a/benchmarks/runners/cli/locate.test.mjs b/benchmarks/runners/cli/locate.test.mjs index 8bc8c3fc..a2d75436 100644 --- a/benchmarks/runners/cli/locate.test.mjs +++ b/benchmarks/runners/cli/locate.test.mjs @@ -2,8 +2,8 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { locateBinary } from "./locate.mjs"; -test("DW_BENCH_BIN override is returned as-is when it exists", () => { - // Point at a file guaranteed to exist: this test file itself. +test("DW_BENCH_BIN override returns an ordinary dw executable as-is", () => { + // Point at a file guaranteed to exist: this test file itself, representing dw. const self = new URL(import.meta.url).pathname; process.env.DW_BENCH_BIN = self; try { @@ -16,7 +16,12 @@ test("DW_BENCH_BIN override is returned as-is when it exists", () => { test("throws an actionable error when the binary is absent", () => { process.env.DW_BENCH_BIN = "/nonexistent/dw-binary-xyz"; try { - assert.throws(() => locateBinary(), /nativeCompile|not found|build/i); + assert.throws( + () => locateBinary(), + (error) => + error.message.includes("Build it with: ./gradlew native-cli:nativeCompile") && + !error.message.includes("-Pbenchmark=true") + ); } finally { delete process.env.DW_BENCH_BIN; } diff --git a/benchmarks/runners/cli/warm.mjs b/benchmarks/runners/cli/warm.mjs deleted file mode 100644 index 7de1af9e..00000000 --- a/benchmarks/runners/cli/warm.mjs +++ /dev/null @@ -1,76 +0,0 @@ -import { spawn } from "node:child_process"; -import { join } from "node:path"; -import { casesForMetric } from "../../lib/manifest.mjs"; -import { computeStats } from "../../lib/stats.mjs"; -import { locateBinary } from "./locate.mjs"; - -function inputArgs(manifest, c) { - const args = []; - for (const [name, inp] of Object.entries(c.inputs ?? {})) { - const file = join(manifest.corpusDir, inp.file); - const charset = inp.charset ?? "utf-8"; - args.push(`--input=${name}=${file}\t${inp.mimeType}\t${charset}`); - } - return args; -} - -/** Spawn dw once in warm mode; resolve the parsed warmMs[] sample array. */ -function warmSamples(bin, manifest, c) { - const scriptPath = join(manifest.corpusDir, c.script); - const warmup = c.iterations?.warmup ?? 10; - const iters = c.iterations?.warm ?? 100; - const args = [ - "--bench-mode=warm", - `--script=${scriptPath}`, - `--warmup=${warmup}`, - `--iters=${iters}`, - ...inputArgs(manifest, c), - ]; - return new Promise((resolve, reject) => { - const child = spawn(bin, args, { - stdio: ["ignore", "pipe", "pipe"], - env: { ...process.env, DW_BENCH: "1" }, - }); - let stdout = ""; - let stderr = ""; - child.stdout.setEncoding("utf-8"); - child.stdout.on("data", (chunk) => (stdout += chunk)); - child.stderr.setEncoding("utf-8"); - child.stderr.on("data", (chunk) => (stderr += chunk)); - child.on("error", reject); - child.on("close", (code) => { - if (code !== 0) { - reject(new Error(`cli warm failed for '${c.id}' (exit ${code})\n${stderr}`)); - return; - } - const jsonLine = stdout.split("\n").filter((l) => l && l !== "READY").pop(); - if (!jsonLine) { - reject(new Error(`cli warm for '${c.id}' printed no result line\n${stderr}`)); - return; - } - let warmMs; - try { - ({ warmMs } = JSON.parse(jsonLine)); - } catch (error) { - reject(new Error(`cli warm for '${c.id}' printed invalid JSON: ${jsonLine}`, { cause: error })); - return; - } - if (!Array.isArray(warmMs) || warmMs.length === 0) { - reject(new Error(`cli warm for '${c.id}' returned no samples\n${stderr}`)); - return; - } - resolve({ warmMs, iters }); - }); - }); -} - -/** @returns {Promise>} */ -export async function runWarm(manifest) { - const bin = locateBinary(); - const rows = []; - for (const c of casesForMetric(manifest, "warm")) { - const { warmMs, iters } = await warmSamples(bin, manifest, c); - rows.push({ id: c.id, metric: "warm", unit: "ms", stats: computeStats(warmMs), iterations: iters }); - } - return rows; -} diff --git a/benchmarks/runners/node/emit.mjs b/benchmarks/runners/node/emit.mjs index 6d18240d..bd687b2c 100644 --- a/benchmarks/runners/node/emit.mjs +++ b/benchmarks/runners/node/emit.mjs @@ -3,7 +3,7 @@ import { join, dirname } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { loadManifest, validateResultIds } from "../../lib/manifest.mjs"; import { gatherEnv } from "../../lib/env.mjs"; -import { loadWrapper } from "./wrapper.mjs"; +import { loadWrapper, resolveDwlibPath } from "./wrapper.mjs"; import { runWarmAndStreaming } from "./warm-bench.mjs"; import { runColdStartAndFirstRun } from "./coldstart.mjs"; @@ -24,7 +24,11 @@ export function buildResult(env, cases) { export async function main() { const manifest = loadManifest(CORPUS); - const env = gatherEnv({ runner: "node-wrapper", runtimeVersion: `node ${process.version}` }); + const env = gatherEnv({ + runner: "node-wrapper", + runtimeVersion: `node ${process.version}`, + dwlibPath: resolveDwlibPath(), + }); // Cold-start / first-run first (fresh processes), then warm/streaming in-process. const coldRows = await runColdStartAndFirstRun(manifest); diff --git a/benchmarks/runners/node/wrapper.mjs b/benchmarks/runners/node/wrapper.mjs index 30ccdeaf..cb3232af 100644 --- a/benchmarks/runners/node/wrapper.mjs +++ b/benchmarks/runners/node/wrapper.mjs @@ -5,23 +5,52 @@ import { fileURLToPath, pathToFileURL } from "node:url"; const __dirname = dirname(fileURLToPath(import.meta.url)); // benchmarks/runners/node -> benchmarks/runners -> benchmarks -> repo root const REPO_ROOT = join(__dirname, "..", "..", ".."); -const WRAPPER_DIST = join(REPO_ROOT, "native-lib", "node", "dist", "index.js"); + +function resolvePackageRoot() { + return process.env.DW_BENCH_NODE_PACKAGE || join(REPO_ROOT, "native-lib", "node"); +} + +export function resolveWrapperPath() { + const wrapperPath = join(resolvePackageRoot(), "dist", "index.js"); + if (existsSync(wrapperPath)) { + return wrapperPath; + } + + if (process.env.DW_BENCH_NODE_PACKAGE) { + throw new Error( + `DW_BENCH_NODE_PACKAGE=${process.env.DW_BENCH_NODE_PACKAGE} does not contain dist/index.js ` + + `(expected an extracted @dataweave/native package)` + ); + } + + throw new Error( + `Node wrapper not built at ${wrapperPath}. ` + + `Run: ./gradlew native-lib:buildNodePackage` + ); +} + +export function resolveDwlibPath() { + const packageRoot = resolvePackageRoot(); + for (const ext of [".dylib", ".so", ".dll"]) { + const candidate = join(packageRoot, "native", `dwlib${ext}`); + if (existsSync(candidate)) { + return candidate; + } + } + return undefined; +} /** * Import the built @dataweave/native wrapper. The wrapper locates dwlib itself * (staged at native-lib/node/native/dwlib.*), so no env var is required here. */ export async function loadWrapper() { - if (!existsSync(WRAPPER_DIST)) { - throw new Error( - `Node wrapper not built at ${WRAPPER_DIST}. ` + - `Run: ./gradlew native-lib:buildNodePackage` - ); - } - const mod = await import(pathToFileURL(WRAPPER_DIST).href); + const wrapperPath = resolveWrapperPath(); + + const mod = await import(pathToFileURL(wrapperPath).href); const api = mod.run ? mod : mod.default; if (!api || typeof api.run !== "function") { - throw new Error(`Wrapper at ${WRAPPER_DIST} did not export a run() function`); + throw new Error(`Wrapper at ${wrapperPath} did not export a run() function`); } return api; } diff --git a/benchmarks/runners/node/wrapper.test.mjs b/benchmarks/runners/node/wrapper.test.mjs new file mode 100644 index 00000000..233fde66 --- /dev/null +++ b/benchmarks/runners/node/wrapper.test.mjs @@ -0,0 +1,80 @@ +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { loadWrapper, resolveDwlibPath, resolveWrapperPath } from "./wrapper.mjs"; + +const tempDirs = []; + +function makeTempDir() { + const dir = join(tmpdir(), `dw-bench-test-${Date.now()}-${Math.random().toString(36).slice(2)}`); + mkdirSync(dir, { recursive: true }); + tempDirs.push(dir); + return dir; +} + +after(() => { + for (const dir of tempDirs) { + try { + rmSync(dir, { recursive: true, force: true }); + } catch {} + } +}); + +test("DW_BENCH_NODE_PACKAGE set to nonexistent dir throws", async () => { + const orig = process.env.DW_BENCH_NODE_PACKAGE; + process.env.DW_BENCH_NODE_PACKAGE = "/nonexistent/test/path"; + try { + await assert.rejects(loadWrapper, /does not contain dist\/index\.js/); + } finally { + if (orig !== undefined) { + process.env.DW_BENCH_NODE_PACKAGE = orig; + } else { + delete process.env.DW_BENCH_NODE_PACKAGE; + } + } +}); + +test("DW_BENCH_NODE_PACKAGE set to valid package dir loads", async () => { + const orig = process.env.DW_BENCH_NODE_PACKAGE; + const packageDir = makeTempDir(); + const distDir = join(packageDir, "dist"); + mkdirSync(distDir, { recursive: true }); + writeFileSync(join(distDir, "index.js"), "export function run() { return null; }"); + + process.env.DW_BENCH_NODE_PACKAGE = packageDir; + try { + const api = await loadWrapper(); + assert.equal(typeof api.run, "function"); + } finally { + if (orig !== undefined) { + process.env.DW_BENCH_NODE_PACKAGE = orig; + } else { + delete process.env.DW_BENCH_NODE_PACKAGE; + } + } +}); + +test("DW_BENCH_NODE_PACKAGE resolves its wrapper and native library", () => { + const orig = process.env.DW_BENCH_NODE_PACKAGE; + const packageDir = makeTempDir(); + const distDir = join(packageDir, "dist"); + const nativeDir = join(packageDir, "native"); + mkdirSync(distDir, { recursive: true }); + mkdirSync(nativeDir, { recursive: true }); + writeFileSync(join(distDir, "index.js"), "export function run() { return null; }"); + writeFileSync(join(nativeDir, "dwlib.dylib"), "fixture native library"); + + process.env.DW_BENCH_NODE_PACKAGE = packageDir; + try { + assert.equal(resolveWrapperPath(), join(distDir, "index.js")); + assert.equal(resolveDwlibPath(), join(nativeDir, "dwlib.dylib")); + } finally { + if (orig !== undefined) { + process.env.DW_BENCH_NODE_PACKAGE = orig; + } else { + delete process.env.DW_BENCH_NODE_PACKAGE; + } + } +}); diff --git a/benchmarks/runners/python/emit.py b/benchmarks/runners/python/emit.py index 862874fb..0e43ae1a 100644 --- a/benchmarks/runners/python/emit.py +++ b/benchmarks/runners/python/emit.py @@ -10,7 +10,7 @@ from env import gather_env from manifest import load_manifest, validate_result_ids from warm_bench import run_warm_and_streaming -from wrapper import load_wrapper +from wrapper import load_wrapper, resolve_dwlib_path # benchmarks/runners/python -> benchmarks _BENCH_DIR = Path(__file__).resolve().parents[2] @@ -46,7 +46,7 @@ def main(): cases = cold_rows + warm_rows validate_result_ids(manifest, [c["id"] for c in cases]) # fail-fast on orphan ids - env = gather_env() + env = gather_env(dwlib_path=resolve_dwlib_path()) result = build_result(env, cases) RESULTS_DIR.mkdir(parents=True, exist_ok=True) diff --git a/benchmarks/runners/python/env.py b/benchmarks/runners/python/env.py index ad6bc960..f9b9140b 100644 --- a/benchmarks/runners/python/env.py +++ b/benchmarks/runners/python/env.py @@ -65,9 +65,10 @@ def _dwlib_path(): return None -def _read_dwlib_build_id(): +def _read_dwlib_build_id(dwlib_path=None): # sha256 over (size + first 64KB), same formula as lib/env.mjs. - p = _dwlib_path() + supplied = Path(dwlib_path) if dwlib_path is not None else None + p = supplied if supplied and supplied.exists() else _dwlib_path() if p and p.exists(): size = p.stat().st_size head = p.read_bytes()[:65536] @@ -78,7 +79,7 @@ def _read_dwlib_build_id(): return "unknown" -def gather_env(): +def gather_env(dwlib_path=None): return { "runner": "python-wrapper", "os": f"{sys.platform}-{_normalize_arch(platform.machine())}", @@ -86,5 +87,5 @@ def gather_env(): "runtimeVersion": f"python {platform.python_version()}", "weaveVersion": _read_weave_version(), "commit": _read_commit(), - "dwlibBuildId": _read_dwlib_build_id(), + "dwlibBuildId": _read_dwlib_build_id(dwlib_path), } diff --git a/benchmarks/runners/python/test_bench.py b/benchmarks/runners/python/test_bench.py index c8fcadda..a1a00ca6 100644 --- a/benchmarks/runners/python/test_bench.py +++ b/benchmarks/runners/python/test_bench.py @@ -6,7 +6,9 @@ from pathlib import Path import hashlib import os +import sys import tempfile +import importlib # benchmarks/runners/python -> benchmarks -> corpus CORPUS = Path(__file__).resolve().parents[2] / "corpus" @@ -111,16 +113,14 @@ def test_dwlib_build_id_formula(self): with tempfile.NamedTemporaryFile(suffix=".dylib", delete=False) as f: f.write(data) path = f.name - os.environ["DATAWEAVE_NATIVE_LIB"] = path try: - e = envmod.gather_env() + e = envmod.gather_env(dwlib_path=Path(path)) size = os.path.getsize(path) h = hashlib.sha256() h.update(str(size).encode()) h.update(data[:65536]) self.assertEqual(e["dwlibBuildId"], "dwlib-" + h.hexdigest()[:8]) finally: - del os.environ["DATAWEAVE_NATIVE_LIB"] os.unlink(path) @@ -130,6 +130,104 @@ def test_load_wrapper_exposes_api(self): for attr in ("DataWeave", "run", "run_transform", "run_streaming"): self.assertTrue(hasattr(api, attr), f"binding missing {attr}") + def test_env_override_missing_dir_raises(self): + with self.assertRaises(RuntimeError): + old_env = os.environ.get("DW_BENCH_PY_SITE") + try: + os.environ["DW_BENCH_PY_SITE"] = "/nonexistent/path/for/test" + wrapper.load_wrapper() + finally: + if old_env is not None: + os.environ["DW_BENCH_PY_SITE"] = old_env + else: + os.environ.pop("DW_BENCH_PY_SITE", None) + + def test_env_override_missing_dataweave_pkg_raises(self): + with tempfile.TemporaryDirectory() as tmpdir: + with self.assertRaises(RuntimeError): + old_env = os.environ.get("DW_BENCH_PY_SITE") + try: + os.environ["DW_BENCH_PY_SITE"] = tmpdir + wrapper.load_wrapper() + finally: + if old_env is not None: + os.environ["DW_BENCH_PY_SITE"] = old_env + else: + os.environ.pop("DW_BENCH_PY_SITE", None) + + def test_env_override_valid_site_loads(self): + with tempfile.TemporaryDirectory() as tmpdir: + dw_pkg = Path(tmpdir) / "dataweave" + dw_pkg.mkdir() + native = dw_pkg / "native" + native.mkdir() + library = native / "dwlib.dylib" + library.write_bytes(b"external dwlib") + (dw_pkg / "__init__.py").write_text( + "class DataWeave: pass\n" + "def run(*a, **k): return None\n" + "def run_transform(*a, **k): return None\n" + "def run_streaming(*a, **k): return None\n" + ) + + old_env = os.environ.get("DW_BENCH_PY_SITE") + old_modules = sys.modules.pop("dataweave", None) + try: + os.environ["DW_BENCH_PY_SITE"] = tmpdir + self.assertEqual(wrapper.resolve_wrapper_site(), Path(tmpdir)) + self.assertEqual(wrapper.resolve_dwlib_path(), library) + api = wrapper.load_wrapper() + self.assertTrue(hasattr(api, "DataWeave")) + self.assertTrue(hasattr(api, "run")) + finally: + if old_env is not None: + os.environ["DW_BENCH_PY_SITE"] = old_env + else: + os.environ.pop("DW_BENCH_PY_SITE", None) + if old_modules is not None: + sys.modules["dataweave"] = old_modules + else: + sys.modules.pop("dataweave", None) + if tmpdir in sys.path: + sys.path.remove(tmpdir) + + def test_override_replaces_cached_local_package(self): + """An override must not reuse an earlier local dataweave import.""" + cached_modules = { + name: module + for name, module in sys.modules.items() + if name == "dataweave" or name.startswith("dataweave.") + } + old_env = os.environ.get("DW_BENCH_PY_SITE") + try: + os.environ.pop("DW_BENCH_PY_SITE", None) + local_api = wrapper.load_wrapper() + self.assertTrue( + Path(local_api.__file__).resolve().is_relative_to(wrapper._SRC.resolve()) + ) + + with tempfile.TemporaryDirectory() as tmpdir: + site = Path(tmpdir) + dw_pkg = site / "dataweave" + dw_pkg.mkdir() + (dw_pkg / "__init__.py").write_text("source = 'override'\n") + + os.environ["DW_BENCH_PY_SITE"] = tmpdir + override_api = wrapper.load_wrapper() + + self.assertTrue( + Path(override_api.__file__).resolve().is_relative_to(site.resolve()) + ) + finally: + if old_env is not None: + os.environ["DW_BENCH_PY_SITE"] = old_env + else: + os.environ.pop("DW_BENCH_PY_SITE", None) + for name in list(sys.modules): + if name == "dataweave" or name.startswith("dataweave."): + del sys.modules[name] + sys.modules.update(cached_modules) + class TestColdstartAggregation(unittest.TestCase): def _manifest(self): diff --git a/benchmarks/runners/python/wrapper.py b/benchmarks/runners/python/wrapper.py index 62713461..26c2452b 100644 --- a/benchmarks/runners/python/wrapper.py +++ b/benchmarks/runners/python/wrapper.py @@ -2,22 +2,56 @@ binding loads the staged dwlib lazily on DataWeave().initialize(), so importing the module itself is dwlib-free.""" +import os import sys +import importlib from pathlib import Path # benchmarks/runners/python -> repo root -> native-lib/python/src _SRC = Path(__file__).resolve().parents[3] / "native-lib" / "python" / "src" +def resolve_wrapper_site(src=None): + # Override for pre-downloaded/published wrapper + if os.environ.get("DW_BENCH_PY_SITE"): + site = Path(os.environ["DW_BENCH_PY_SITE"]) + if not site.is_dir() or not (site / "dataweave" / "__init__.py").exists(): + raise RuntimeError( + f"DW_BENCH_PY_SITE={site} does not contain dataweave/__init__.py " + f"(expected a site-packages-style directory)" + ) + return site + + return Path(src) if src is not None else _SRC + + +def resolve_dwlib_path(src=None): + root = resolve_wrapper_site(src) + for extension in (".dylib", ".so", ".dll"): + candidate = root / "dataweave" / "native" / f"dwlib{extension}" + if candidate.exists(): + return candidate + return None + + def load_wrapper(src=None): - src = Path(src) if src is not None else _SRC - if str(src) not in sys.path: - sys.path.insert(0, str(src)) + site = resolve_wrapper_site(src) + if str(site) not in sys.path: + sys.path.insert(0, str(site)) + if os.environ.get("DW_BENCH_PY_SITE"): + selected_site = site.resolve() + for name, module in list(sys.modules.items()): + if name != "dataweave" and not name.startswith("dataweave."): + continue + module_file = getattr(module, "__file__", None) + if module_file and not Path(module_file).resolve().is_relative_to(selected_site): + del sys.modules[name] + importlib.invalidate_caches() try: import dataweave except ImportError as e: raise RuntimeError( - f"DataWeave Python binding not importable from {src}. " + f"DataWeave Python binding not importable from {site}. " f"Run: ./gradlew native-lib:stagePythonNativeLib ({e})" ) return dataweave diff --git a/docs/superpowers/specs/2026-07-22-native-lib-benchmarks-design.md b/docs/superpowers/specs/2026-07-22-native-lib-benchmarks-design.md index be5d6673..693076bc 100644 --- a/docs/superpowers/specs/2026-07-22-native-lib-benchmarks-design.md +++ b/docs/superpowers/specs/2026-07-22-native-lib-benchmarks-design.md @@ -4,6 +4,12 @@ **Status:** Approved (design) **Scope of first deliverable:** Node runner + common JSON schema + report script. Python and Scala-engine runners are follow-up specs; the schema and corpus are designed to accommodate them without change. +> **Superseded by current implementation:** This is the original harness design record. +> Node, Python, engine, and CLI runners are now implemented, and task wiring has +> evolved. Use [`benchmarks/README.md`](../../benchmarks/README.md) and the +> current runner sources for operational documentation; retain this document for +> its original decisions and rationale. + ## Purpose Benchmark the DataWeave native-lib wrappers to serve, from one harness: diff --git a/docs/superpowers/specs/2026-07-23-engine-runner-design.md b/docs/superpowers/specs/2026-07-23-engine-runner-design.md index aa1081cc..718b7769 100644 --- a/docs/superpowers/specs/2026-07-23-engine-runner-design.md +++ b/docs/superpowers/specs/2026-07-23-engine-runner-design.md @@ -4,6 +4,12 @@ **Status:** Approved (design) **Parent spec:** [`2026-07-22-native-lib-benchmarks-design.md`](./2026-07-22-native-lib-benchmarks-design.md) — this resolves the three JVM-specific decisions that spec deferred to the engine runner. +> **Superseded by current implementation:** This document records the original +> JVM runner design. For current task wiring, runner layout, and usage, use +> [`benchmarks/README.md`](../../benchmarks/README.md) and +> [`benchmarks/runners/engine/`](../../benchmarks/runners/engine/). Retain this +> document for its original decisions and rationale. + ## Purpose Build the **engine runner** — the JVM baseline the native-lib wrappers are compared against. It drives the DataWeave engine over the **same shared corpus** the Node runner consumes and emits the **same JSON schema**, so `report/report.mjs` joins the results and produces the headline delta: **native-image wrappers vs. the JVM engine**. diff --git a/docs/superpowers/specs/2026-07-23-python-runner-design.md b/docs/superpowers/specs/2026-07-23-python-runner-design.md index e2e57371..566bdd7e 100644 --- a/docs/superpowers/specs/2026-07-23-python-runner-design.md +++ b/docs/superpowers/specs/2026-07-23-python-runner-design.md @@ -5,6 +5,12 @@ **Parent spec:** [`2026-07-22-native-lib-benchmarks-design.md`](./2026-07-22-native-lib-benchmarks-design.md) — the corpus/schema/report contract every runner shares, which lists the Python runner as an explicit follow-up. **Sibling precedent:** [`2026-07-23-engine-runner-design.md`](./2026-07-23-engine-runner-design.md) — the closest structural template; this mirrors its self-contained-emit-with-parity-test playbook, one language over. +> **Superseded by current implementation:** This document records the original +> Python runner design. For current task wiring, external-artifact overrides, and +> test coverage, use [`benchmarks/README.md`](../../benchmarks/README.md) and +> [`benchmarks/runners/python/`](../../benchmarks/runners/python/). Retain this +> document for its original decisions and rationale. + ## Purpose Build the **Python runner** — the third benchmark surface, alongside the Node wrapper and the JVM engine baseline. It drives the DataWeave **Python** binding (`native-lib/python`, which wraps the same staged `dwlib` the Node wrapper does) over the **same shared corpus** and emits the **same JSON schema**, so `report/report.mjs` joins its results and produces cross-binding deltas: **Python wrapper vs. Node wrapper vs. JVM engine** — all at the same `weaveVersion`, all through the aggregator (`benchmarkCompare`). diff --git a/docs/superpowers/specs/2026-07-27-cli-benchmark-runner-design.md b/docs/superpowers/specs/2026-07-27-cli-benchmark-runner-design.md index eb2c615f..d9c34579 100644 --- a/docs/superpowers/specs/2026-07-27-cli-benchmark-runner-design.md +++ b/docs/superpowers/specs/2026-07-27-cli-benchmark-runner-design.md @@ -2,6 +2,13 @@ _2026-07-27_ +> **Superseded:** This document records the original CLI runner design. The +> current design is [CLI End-to-End Benchmark](2026-08-10-cli-end-to-end-benchmark-design.md). +> For current task wiring, including `DW_BENCH_BIN`, and runner usage, use +> [`benchmarks/README.md`](../../benchmarks/README.md) and +> [`benchmarks/runners/cli/`](../../benchmarks/runners/cli/). Retain this +> document for its original decisions and rationale. + ## Goal Add a fourth runner to the `benchmarks/` harness that measures the **`dw` native diff --git a/docs/superpowers/specs/2026-08-10-cli-end-to-end-benchmark-design.md b/docs/superpowers/specs/2026-08-10-cli-end-to-end-benchmark-design.md new file mode 100644 index 00000000..c5a472ac --- /dev/null +++ b/docs/superpowers/specs/2026-08-10-cli-end-to-end-benchmark-design.md @@ -0,0 +1,81 @@ +# CLI End-to-End Benchmark — Design + +**Date:** 2026-08-10 +**Status:** Implemented + +## Goal + +Make the CLI benchmark measure the customer-visible `dw run` command rather +than an in-process benchmark harness. The CLI runner emits only `first-run`, +defined for this runner as whole-command latency. + +## Metric Semantics + +For the CLI runner, `first-run` is wall-clock time from just before spawning a +normal `dw run` process until its successful exit. It includes process launch, +native-image load, CLI argument parsing, `NativeRuntime` construction, script +compilation, execution, and output writing. + +This differs intentionally from the in-process `first-run` emitted by the +Node, Python, and engine runners. The README and report output must state the +distinction so cross-runner readers do not interpret the values as equivalent +microbenchmarks. + +The CLI emits no `cold-start`, `warm`, or `streaming` rows. The shared schema +continues to support those metrics for the other runners. + +## Runner Architecture + +`benchmarks/runners/cli/` becomes a normal-command parent only: + +- Read the shared manifest and select cases declaring `first-run`. +- For every configured sample, spawn the selected `dw` binary using its normal + `run` command with the corpus script and declared inputs. Input MIME types are + inferred by the existing CLI from file extensions; the current UTF-16 XML + corpus input has been verified through this public path and remains included. +- Capture stdout and stderr, fail the sample on nonzero exit, and measure + spawn-to-exit elapsed time with `process.hrtime.bigint()`. +- Aggregate samples with the shared `computeStats` helper and emit standard + flat result rows using metric `first-run` and unit `ms`. + +The runner must use the same production binary that customers invoke. +`DW_BENCH_BIN` remains an optional path override, but it no longer requires a +benchmark-enabled artifact. + +## Removed Components + +Remove all benchmark-only behavior from `native-cli`: + +- Generated `BenchmarkMode` source and its Gradle generation task/wiring. +- `DWCLI` dispatch based on `BenchmarkMode` and `DW_BENCH`. +- `BenchmarkHarness.scala` and its test suite. +- Build-time `-Pbenchmark=true` requirement for compiling a benchmark harness. + +Remove the CLI runner's `coldstart.mjs` and `warm.mjs`, including their +`coldfirst`, `warm`, and `READY` protocol handling. Replace them with one +normal-command sampling module and focused parent-level tests using a fake +child process; tests must not require a native binary. + +## Gradle and Documentation + +`native-cli:benchmarkCli` remains an opt-in, aggregator-registered task. It +continues to depend on `nativeCompile` when `DW_BENCH_BIN` is absent and skips +that dependency when the override is set. It no longer relies on +`-Pbenchmark=true` to make the selected binary capable of benchmark execution; +the property gates task execution only. + +Update `benchmarks/README.md` to document the CLI's end-to-end `first-run` +semantics and its absence of `cold-start`, `warm`, and `streaming`. Update +report text/labels to distinguish CLI end-to-end `first-run` from in-process +first-run results. Mark the prior CLI benchmark design as superseded by this +document. + +## Testing + +- Unit-test command construction, successful sample parsing, nonzero-exit + handling, and timing-row aggregation through injected/fake child execution. +- Verify the CLI emitter produces only `first-run` rows from a representative + manifest fixture. +- Run the dependency-free Node benchmark-harness test task. +- Run Gradle dry runs with and without `DW_BENCH_BIN` to confirm the existing + dependency behavior remains intact. diff --git a/docs/superpowers/specs/2026-08-10-external-benchmark-artifacts-design.md b/docs/superpowers/specs/2026-08-10-external-benchmark-artifacts-design.md new file mode 100644 index 00000000..6cb7a976 --- /dev/null +++ b/docs/superpowers/specs/2026-08-10-external-benchmark-artifacts-design.md @@ -0,0 +1,58 @@ +# External benchmark artifacts — design + +**Date:** 2026-08-10 +**Status:** Approved + +## Goal + +Allow the benchmark runners to execute against existing Node and Python wrapper +artifacts without triggering local native builds. Remove the redundant legacy +Node-only `native-lib:benchmark` task. + +## Supported tasks + +- `native-lib:benchmarkNode` remains the Node runner task and emits a result JSON. +- `native-lib:benchmarkPython` remains the Python runner task and emits a result + JSON. +- `benchmarkCompare` continues to invoke registered runner tasks and render one + comparison report. +- Remove `native-lib:benchmark`; it duplicates the Node runner while also rendering + a report, unlike the runner-task contract. + +## Artifact selection + +- Without an override, `benchmarkNode` depends on `buildNodePackage` and + `benchmarkPython` depends on `stagePythonNativeLib`, preserving current local + build behavior. +- With `DW_BENCH_NODE_PACKAGE`, `benchmarkNode` must not depend on + `buildNodePackage`; it loads the extracted package at that path. +- With `DW_BENCH_PY_SITE`, `benchmarkPython` must not depend on + `stagePythonNativeLib`; it imports the site-packages-style directory at that + path. +- Invalid overrides fail immediately and never fall back to a local artifact. + +## Provenance + +Each result must identify the native library actually selected by its runner. +The existing `dwlibBuildId` formula remains unchanged: hash the file size and its +first 64 KiB. Node and Python environment collection resolve the library from the +configured external artifact when an override is active; otherwise they use the +existing local staging paths. + +## Documentation and regression fixes + +- Remove the legacy `native-lib:benchmark` invocation from benchmark documentation. +- Document external-artifact use with `benchmarkNode`, `benchmarkPython`, and + `benchmarkCompare`. +- Correct the CLI documentation to state that `DW_BENCH_BIN` accepts a prebuilt, + benchmark-enabled binary. +- Correct the executable Python example to use the `InputValue.mime_type` keyword. + +## Testing + +- Extend focused Node and Python helper tests to verify external library-path + resolution and corresponding `dwlibBuildId` attribution. +- Add Gradle configuration-level coverage where practical to verify override paths + do not attach local artifact build dependencies. +- Run the dependency-free Node and Python benchmark-harness test tasks and Gradle + task discovery/help checks for the removed legacy task. diff --git a/native-cli/build.gradle b/native-cli/build.gradle index 05c066ad..63757920 100644 --- a/native-cli/build.gradle +++ b/native-cli/build.gradle @@ -70,38 +70,10 @@ task genVersions() { outputPrinter.close() } -def genJavaDirectory = new File("$project.buildDir/genjava") - -task genBenchmarkMode() { - def enabled = project.findProperty('benchmark')?.toString()?.toBoolean() == true - def benchmarkMode = new File(genJavaDirectory, "org/mule/weave/cli/BenchmarkMode.java") - def parentFile = benchmarkMode.getParentFile() - if (!parentFile.exists()) { - parentFile.mkdirs() - } - final PrintWriter outputPrinter = new PrintWriter(new FileWriter(benchmarkMode)) - outputPrinter.println("package org.mule.weave.cli;") - outputPrinter.println() - outputPrinter.println("// GENERATED by genBenchmarkMode — do not edit.") - outputPrinter.println("// ENABLED is true only when built with -Pbenchmark=true; native-image") - outputPrinter.println("// folds the benchmark branch away as dead code when this is false.") - outputPrinter.println("public final class BenchmarkMode {") - outputPrinter.println(" private BenchmarkMode() {}") - outputPrinter.println(" public static final boolean ENABLED = " + enabled + ";") - outputPrinter.println("}") - outputPrinter.close() -} - - defaultTasks += genVersions compileScala { dependsOn genVersions - dependsOn genBenchmarkMode -} - -compileJava { - dependsOn genBenchmarkMode } // Merging Service Files @@ -193,13 +165,14 @@ tasks.compileJava.classpath += files(sourceSets.main.scala.classesDirectory) // The CLI runner as an aggregator-registered runner: emits its result file but // does NOT render the report (the root :benchmarkCompare renders once over all // runners). Tagged `benchmarkRunner` so :benchmarkCompare discovers it automatically. -// Requires the bench-enabled binary — nativeCompile must run with -Pbenchmark=true so -// BenchmarkMode.ENABLED is true and BenchmarkHarness is reachable in dw. +// Benchmarks the ordinary production binary. tasks.register('benchmarkCli', Exec) { onlyIf { project.findProperty('benchmark')?.toString()?.toBoolean() == true } ext.benchmarkRunner = true - dependsOn tasks.named('nativeCompile') + if (!System.getenv('DW_BENCH_BIN')) { + dependsOn tasks.named('nativeCompile') + } workingDir("${rootDir}/benchmarks") def script = 'node corpus/gen-inputs.mjs && node runners/cli/emit.mjs' @@ -208,4 +181,4 @@ tasks.register('benchmarkCli', Exec) { } else { commandLine('bash', '-c', script) } -} \ No newline at end of file +} diff --git a/native-cli/src/main/java/org/mule/weave/cli/DWCLI.java b/native-cli/src/main/java/org/mule/weave/cli/DWCLI.java index 6d430a0c..feb7efe0 100644 --- a/native-cli/src/main/java/org/mule/weave/cli/DWCLI.java +++ b/native-cli/src/main/java/org/mule/weave/cli/DWCLI.java @@ -30,14 +30,6 @@ public class DWCLI { public static void main(String[] args) { - // Benchmark dispatch: only reachable in a build made with -Pbenchmark=true. - // The outer compile-time constant lets javac remove this block from production. - if (BenchmarkMode.ENABLED) { - if (System.getenv("DW_BENCH") != null) { - org.mule.weave.dwnative.benchmark.BenchmarkHarness.main(args); - return; - } - } new DWCLI().run(args, DefaultConsole$.MODULE$); } diff --git a/native-cli/src/main/scala/org/mule/weave/dwnative/benchmark/BenchmarkHarness.scala b/native-cli/src/main/scala/org/mule/weave/dwnative/benchmark/BenchmarkHarness.scala deleted file mode 100644 index 7b5e9cba..00000000 --- a/native-cli/src/main/scala/org/mule/weave/dwnative/benchmark/BenchmarkHarness.scala +++ /dev/null @@ -1,127 +0,0 @@ -package org.mule.weave.dwnative.benchmark - -import org.mule.weave.dwnative.NativeRuntime -import org.mule.weave.dwnative.WeaveExecutionResult -import org.mule.weave.dwnative.cli.DefaultConsole -import org.mule.weave.dwnative.utils.DataWeaveUtils -import org.mule.weave.v2.runtime.BindingValue -import org.mule.weave.v2.runtime.ScriptingBindings - -import java.io.{ File, OutputStream, PrintStream } -import java.nio.charset.Charset -import java.nio.file.Files - -final case class BenchInput(name: String, file: String, mimeType: String, charset: String) -final case class BenchArgs(mode: String, scriptFile: String, inputs: Seq[BenchInput], warmup: Int, iters: Int) - -/** Corpus-agnostic in-binary benchmark harness. Reachable only in a build made with - * -Pbenchmark=true (guarded by BenchmarkMode.ENABLED in DWCLI); native-image folds it - * out of a production dw. Prints "READY" the instant one NativeRuntime is constructed, - * then a single JSON line of timings. Parent (benchmarks/runners/cli) measures cold-start - * as spawn->READY wall-clock. */ -object BenchmarkHarness { - - /** Discards bytes; used as the transform write sink so we never touch real stdout. */ - private final class DiscardStream extends OutputStream { - override def write(b: Int): Unit = () - override def write(b: Array[Byte]): Unit = () - override def write(b: Array[Byte], off: Int, len: Int): Unit = () - } - - private def nowNs(): Long = System.nanoTime() - private def msSince(startNs: Long): Double = (System.nanoTime() - startNs) / 1e6 - - def parseArgs(args: Array[String]): BenchArgs = { - var mode = "" - var script = "" - val inputs = scala.collection.mutable.ArrayBuffer[BenchInput]() - var warmup = 0 - var iters = 100 - args.foreach { arg => - val eq = arg.indexOf('=') - val key = if (eq >= 0) arg.substring(0, eq) else arg - val value = if (eq >= 0) arg.substring(eq + 1) else "" - key match { - case "--bench-mode" => mode = value - case "--script" => script = value - case "--warmup" => warmup = value.toInt - case "--iters" => iters = value.toInt - case "--input" => - // value = =\t[\t] - val nameSep = value.indexOf('=') - val name = value.substring(0, nameSep) - val rest = value.substring(nameSep + 1) - val parts = rest.split("\t", 3) - val file = parts(0) - val mimeType = parts(1) - val charset = if (parts.length > 2 && parts(2).nonEmpty) parts(2) else "utf-8" - inputs += BenchInput(name, file, mimeType, charset) - case _ => throw new RuntimeException(s"unknown bench arg: $arg") - } - } - if (mode.isEmpty) throw new RuntimeException("--bench-mode is required") - if (script.isEmpty) throw new RuntimeException("--script is required") - BenchArgs(mode, script, inputs.toSeq, warmup, iters) - } - - private def newRuntime(): NativeRuntime = { - val console = DefaultConsole.enableSilent() - val utils = new DataWeaveUtils(console) - new NativeRuntime(utils.getLibPathHome(), Array.empty[File], console, None) - } - - private def readScript(a: BenchArgs): String = - new String(Files.readAllBytes(new File(a.scriptFile).toPath), java.nio.charset.StandardCharsets.UTF_8) - - private def bindings(a: BenchArgs): ScriptingBindings = { - val b = new ScriptingBindings() - a.inputs.foreach { in => - val bytes = Files.readAllBytes(new File(in.file).toPath) - val bv = new BindingValue(bytes, Some(in.mimeType), Map.empty[String, Any], Charset.forName(in.charset)) - b.addBinding(in.name, bv) - } - b - } - - private def assertOk(r: WeaveExecutionResult): Unit = - if (!r.success()) throw new RuntimeException("run failed: " + r.result()) - - def runColdFirst(a: BenchArgs, out: PrintStream, sink: OutputStream): Unit = { - val script = readScript(a) - val b = bindings(a) - val rt = newRuntime() // engine init — measured externally as cold-start - out.print("READY\n"); out.flush() - val start = nowNs() - assertOk(rt.run(script, "bench", b, sink, "application/json", None)) - val firstRunMs = msSince(start) - out.print("{\"firstRunMs\":" + firstRunMs + "}\n") - } - - def runWarm(a: BenchArgs, out: PrintStream, sink: OutputStream): Unit = { - val script = readScript(a) - val b = bindings(a) - val rt = newRuntime() - out.print("READY\n"); out.flush() - var i = 0 - while (i < a.warmup) { assertOk(rt.run(script, "bench", b, sink, "application/json", None)); i += 1 } - val samples = new Array[Double](a.iters) - i = 0 - while (i < a.iters) { - val start = nowNs() - assertOk(rt.run(script, "bench", b, sink, "application/json", None)) - samples(i) = msSince(start) - i += 1 - } - out.print("{\"warmMs\":[" + samples.mkString(",") + "]}\n") - } - - def main(args: Array[String]): Unit = { - val a = parseArgs(args) - val sink = new DiscardStream() - a.mode match { - case "coldfirst" => runColdFirst(a, System.out, sink) - case "warm" => runWarm(a, System.out, sink) - case other => throw new RuntimeException(s"unknown --bench-mode: $other") - } - } -} diff --git a/native-cli/src/test/scala/org/mule/weave/dwnative/benchmark/BenchmarkHarnessTest.scala b/native-cli/src/test/scala/org/mule/weave/dwnative/benchmark/BenchmarkHarnessTest.scala deleted file mode 100644 index 81342839..00000000 --- a/native-cli/src/test/scala/org/mule/weave/dwnative/benchmark/BenchmarkHarnessTest.scala +++ /dev/null @@ -1,104 +0,0 @@ -package org.mule.weave.dwnative.benchmark - -import org.scalatest.freespec.AnyFreeSpec -import org.scalatest.matchers.should.Matchers - -import java.io.{ ByteArrayOutputStream, File, PrintStream } -import java.nio.charset.StandardCharsets -import java.nio.file.Files - -class BenchmarkHarnessTest extends AnyFreeSpec with Matchers { - - private def tmp(suffix: String, content: String): File = { - val f = File.createTempFile("bench", suffix) - f.deleteOnExit() - Files.write(f.toPath, content.getBytes(StandardCharsets.UTF_8)) - f - } - - private def capture(fn: PrintStream => Unit): String = { - val buf = new ByteArrayOutputStream() - val ps = new PrintStream(buf, true, "UTF-8") - fn(ps) - new String(buf.toByteArray, StandardCharsets.UTF_8) - } - - "parseArgs" - { - "parses coldfirst mode with one input" in { - val a = BenchmarkHarness.parseArgs(Array( - "--bench-mode=coldfirst", - "--script=/tmp/x.dwl", - "--input=payload=/tmp/p.json\tapplication/json\tutf-8")) - a.mode shouldBe "coldfirst" - a.scriptFile shouldBe "/tmp/x.dwl" - a.inputs should have size 1 - a.inputs.head shouldBe BenchInput("payload", "/tmp/p.json", "application/json", "utf-8") - } - - "parses warm mode with warmup and iters" in { - val a = BenchmarkHarness.parseArgs(Array( - "--bench-mode=warm", "--script=/tmp/x.dwl", "--warmup=5", "--iters=30")) - a.mode shouldBe "warm" - a.warmup shouldBe 5 - a.iters shouldBe 30 - a.inputs shouldBe empty - } - - "handles a mimeType-only input (charset defaults to utf-8)" in { - val a = BenchmarkHarness.parseArgs(Array( - "--bench-mode=coldfirst", "--script=/tmp/x.dwl", - "--input=payload=/tmp/p.json\tapplication/json")) - a.inputs.head.charset shouldBe "utf-8" - } - } - - "runColdFirst" - { - "emits READY then a single firstRunMs JSON line, output not on the stream" in { - val script = tmp(".dwl", "output application/json --- payload.a + 1") - val input = tmp(".json", "{\"a\": 41}") - val a = BenchArgs("coldfirst", script.getAbsolutePath, - Seq(BenchInput("payload", input.getAbsolutePath, "application/json", "utf-8")), 0, 0) - val sink = new ByteArrayOutputStream() - val stdout = capture(ps => BenchmarkHarness.runColdFirst(a, ps, sink)) - val lines = stdout.split("\n").filter(_.nonEmpty) - lines.head shouldBe "READY" - lines.last should include ("firstRunMs") - lines.count(_.contains("firstRunMs")) shouldBe 1 - // The transformed "42" went to the sink, NOT to stdout. - new String(sink.toByteArray, StandardCharsets.UTF_8).trim shouldBe "42" - } - } - - "runWarm" - { - "emits READY then a warmMs array of length iters" in { - val script = tmp(".dwl", "output application/json --- payload.a + 1") - val input = tmp(".json", "{\"a\": 41}") - val a = BenchArgs("warm", script.getAbsolutePath, - Seq(BenchInput("payload", input.getAbsolutePath, "application/json", "utf-8")), 1, 3) - val stdout = capture(ps => BenchmarkHarness.runWarm(a, ps, new ByteArrayOutputStream())) - val json = stdout.split("\n").filter(_.contains("warmMs")).head - json should include ("warmMs") - // 3 comma-separated samples -> 2 commas inside the array - json.count(_ == ',') shouldBe 2 - } - } - - "a failing script throws (non-zero exit path)" in { - val script = tmp(".dwl", "output application/json --- 1 / 0") - val input = tmp(".json", "{}") - val a = BenchArgs("coldfirst", script.getAbsolutePath, - Seq(BenchInput("payload", input.getAbsolutePath, "application/json", "utf-8")), 0, 0) - an [RuntimeException] should be thrownBy - BenchmarkHarness.runColdFirst(a, capturePs(), new ByteArrayOutputStream()) - } - - private def capturePs(): PrintStream = new PrintStream(new ByteArrayOutputStream(), true, "UTF-8") - - "BenchmarkMode.ENABLED" - { - "is false in a normal (non -Pbenchmark) build" in { - // Tests run without -Pbenchmark, so the generated constant must be false — - // proving the harness is dead code / stripped from a production image. - org.mule.weave.cli.BenchmarkMode.ENABLED shouldBe false - } - } -} diff --git a/native-lib/build.gradle b/native-lib/build.gradle index 8a373d98..436f6066 100644 --- a/native-lib/build.gradle +++ b/native-lib/build.gradle @@ -242,23 +242,6 @@ tasks.register('nodeTest', Exec) { } } -tasks.register('benchmark', Exec) { - // Opt-in only: skipped unless -Pbenchmark=true. Never part of build/test. - // Standalone Node-only convenience: emit + render the report by itself. - // For the cross-runner comparison use the root :benchmarkCompare task. - onlyIf { project.findProperty('benchmark')?.toString()?.toBoolean() == true } - - dependsOn tasks.named('buildNodePackage') - workingDir("${rootDir}/benchmarks") - - def script = 'node corpus/gen-inputs.mjs && node runners/node/emit.mjs && node report/report.mjs results/*.json' - if (System.getProperty('os.name').toLowerCase().contains('windows')) { - commandLine('cmd', '/c', script) - } else { - commandLine('bash', '-c', script) - } -} - // The Node runner as an aggregator-registered runner: emits its result file but // does NOT render the report (the root :benchmarkCompare renders once over all // runners). Tagged `benchmarkRunner` so :benchmarkCompare discovers it automatically. @@ -266,7 +249,9 @@ tasks.register('benchmarkNode', Exec) { onlyIf { project.findProperty('benchmark')?.toString()?.toBoolean() == true } ext.benchmarkRunner = true - dependsOn tasks.named('buildNodePackage') + if (!System.getenv('DW_BENCH_NODE_PACKAGE')) { + dependsOn tasks.named('buildNodePackage') + } workingDir("${rootDir}/benchmarks") // Generate the shared input (idempotent, deterministic) then emit; no report here. @@ -287,7 +272,9 @@ tasks.register('benchmarkPython', Exec) { onlyIf { project.findProperty('benchmark')?.toString()?.toBoolean() == true } ext.benchmarkRunner = true - dependsOn tasks.named('stagePythonNativeLib') + if (!System.getenv('DW_BENCH_PY_SITE')) { + dependsOn tasks.named('stagePythonNativeLib') + } workingDir("${rootDir}/benchmarks") // Generate the shared input via the Node generator (idempotent, deterministic, @@ -318,8 +305,8 @@ tasks.register('benchmarkJsUnitTest', Exec) { } workingDir("${rootDir}/benchmarks") def files = 'lib/stats.test.mjs lib/manifest.test.mjs lib/env.test.mjs ' + - 'report/report.test.mjs runners/node/emit.test.mjs ' + - 'runners/cli/locate.test.mjs runners/cli/emit.test.mjs' + 'report/report.test.mjs runners/node/emit.test.mjs runners/node/wrapper.test.mjs ' + + 'runners/cli/locate.test.mjs runners/cli/first-run.test.mjs runners/cli/emit.test.mjs' def script = 'node --test ' + files if (System.getProperty('os.name').toLowerCase().contains('windows')) { commandLine('cmd', '/c', script) diff --git a/native-lib/example_dataweave_module.py b/native-lib/example_dataweave_module.py index d1740a25..7875986d 100755 --- a/native-lib/example_dataweave_module.py +++ b/native-lib/example_dataweave_module.py @@ -25,59 +25,60 @@ def example_simple_functions(): # Simple script execution print("\n[*] Simple arithmetic:") script = "2 + 2" - result = dataweave.run_script(script) + result = dataweave.run(script) ok = assert_result(script, result, "4") and ok print("\n[*] Square root:") script = "sqrt(144)" - result = dataweave.run_script(script) + result = dataweave.run(script) ok = assert_result(script, result, "12") and ok print("\n[*] Array operations:") script = "[1, 2, 3] map $ * 2" - result = dataweave.run_script(script) + result = dataweave.run(script) ok = assert_result(script, result, "[\n 2, \n 4, \n 6\n]") and ok print("\n[*] String operations:") script = "upper('hello world')" - result = dataweave.run_script(script) + result = dataweave.run(script) ok = assert_result(script, result, '"HELLO WORLD"') and ok # Script with inputs (simple values - auto-converted) print("\n[*] Script with inputs (auto-converted):") script = "num1 + num2" - result = dataweave.run_script(script, {"num1": 25, "num2": 17}) + result = dataweave.run(script, {"num1": 25, "num2": 17}) ok = assert_result(script, result, "42") and ok # Script with complex inputs print("\n[*] Script with complex object:") script = "payload.name" - result = dataweave.run_script(script, {"payload": {"content": '{"name": "John", "age": 30}', "mimeType": "application/json"}}) + result = dataweave.run(script, {"payload": {"content": '{"name": "John", "age": 30}', "mimeType": "application/json"}}) ok = assert_result(script, result, '"John"') and ok # Script with mixed input types print("\n[*] Script with mixed input types:") script = "greeting ++ ' ' ++ payload.name" - result = dataweave.run_script(script, {"greeting": "Hello", "payload": {"content": '{"name": "Alice", "role": "Developer"}', "mimeType": "application/json"}}) + result = dataweave.run(script, {"greeting": "Hello", "payload": {"content": '{"name": "Alice", "role": "Developer"}', "mimeType": "application/json"}}) ok = assert_result(script, result, '"Hello Alice"') and ok # Binary output print("\n[*] Binary output:") script = "output application/octet-stream\n---\ndw::core::Binaries::fromBase64(\"holamund\")" - result = dataweave.run_script(script) + result = dataweave.run(script) ok = assert_result(script, result, "holamund") and ok # Script with InputValue print("\n[*] Inputs:") input_value = dataweave.InputValue( content="1234567", - mimeType="application/csv", + mime_type="application/csv", properties={"header": False, "separator": "4"} ) script = "in0.column_1[0]" - result = dataweave.run_script(script, {"in0": input_value}) + result = dataweave.run(script, {"in0": input_value}) ok = assert_result(script, result, '"567"') and ok + # Cleanup when done dataweave.cleanup() print("\n[OK] Cleanup completed") @@ -136,11 +137,11 @@ def example_explicit_format(): ok = True script = "payload.message" - result = dataweave.run_script(script, {"payload": {"content": '{"message": "Hello from JSON!", "value": 42}', "mimeType": "application/json"}}) + result = dataweave.run(script, {"payload": {"content": '{"message": "Hello from JSON!", "value": 42}', "mimeType": "application/json"}}) ok = assert_result(script, result, '"Hello from JSON!"') and ok script = "payload.value + offset" - result = dataweave.run_script(script, {"payload": {"content": '{"value": 100}', "mimeType": "application/json"}, "offset": 50}) + result = dataweave.run(script, {"payload": {"content": '{"value": 100}', "mimeType": "application/json"}, "offset": 50}) ok = assert_result(script, result, "150") and ok return ok @@ -154,7 +155,7 @@ def example_error_handling(): try: print("\n[*] Invalid script (will show error):") - result = dataweave.run_script("invalid syntax here", {}) + result = dataweave.run("invalid syntax here", {}) print(f" Result: {result} {'[OK]' if result.success == False else '[FAIL]'}") except dataweave.DataWeaveLibraryNotFoundError as e: diff --git a/native-lib/python/tests/test_dataweave_module.py b/native-lib/python/tests/test_dataweave_module.py index a4df9925..959834b3 100755 --- a/native-lib/python/tests/test_dataweave_module.py +++ b/native-lib/python/tests/test_dataweave_module.py @@ -11,6 +11,22 @@ import dataweave +def test_input_value_mime_type_constructor(): + """Test InputValue accepts the public mime_type constructor keyword.""" + print("Testing InputValue mime_type constructor...") + try: + value = dataweave.InputValue( + content="1234567", + mime_type="application/csv", + properties={"header": False, "separator": "4"}, + ) + assert value.mime_type == "application/csv" + print("[OK] InputValue mime_type constructor works") + return True + except Exception as e: + print(f"[FAIL] InputValue mime_type constructor failed: {e}") + return False + def test_basic(): """Test basic functionality""" print("Testing basic script execution...") @@ -473,6 +489,7 @@ def main(): try: results = [] + results.append(test_input_value_mime_type_constructor()) results.append(test_basic()) results.append(test_with_inputs()) results.append(test_context_manager())