From 645a7c5c64561ff72fe65ff52d5e8d0866a95309 Mon Sep 17 00:00:00 2001 From: Adwait Kumar Singh Date: Sat, 22 Aug 2026 05:25:19 +0530 Subject: [PATCH] Add ops/CPU-sec metric to benchmarks --- benchmarks/benchmark-commons/build.gradle.kts | 15 +++ .../benchmarks/OpsPerCpuSecondProfiler.java | 97 +++++++++++++++++++ .../java/benchmarks/ProcessCpuTime.java | 59 +++++++++++ .../OpsPerCpuSecondProfilerTest.java | 24 +++++ .../java/benchmarks/ProcessCpuTimeTest.java | 27 ++++++ benchmarks/e2e-benchmarks/README.md | 5 + benchmarks/e2e-benchmarks/build.gradle.kts | 16 +++ .../java/benchmarks/e2e/BenchmarkSupport.java | 26 ++++- benchmarks/serde-benchmarks/README.md | 7 +- benchmarks/serde-benchmarks/build.gradle.kts | 17 ++++ .../benchmarks/serde/JmhResultConverter.java | 21 +++- gradle/libs.versions.toml | 1 + scripts/run-remote-benchmarks.sh | 20 +++- settings.gradle.kts | 1 + 14 files changed, 323 insertions(+), 13 deletions(-) create mode 100644 benchmarks/benchmark-commons/build.gradle.kts create mode 100644 benchmarks/benchmark-commons/src/main/java/software/amazon/smithy/java/benchmarks/OpsPerCpuSecondProfiler.java create mode 100644 benchmarks/benchmark-commons/src/main/java/software/amazon/smithy/java/benchmarks/ProcessCpuTime.java create mode 100644 benchmarks/benchmark-commons/src/test/java/software/amazon/smithy/java/benchmarks/OpsPerCpuSecondProfilerTest.java create mode 100644 benchmarks/benchmark-commons/src/test/java/software/amazon/smithy/java/benchmarks/ProcessCpuTimeTest.java diff --git a/benchmarks/benchmark-commons/build.gradle.kts b/benchmarks/benchmark-commons/build.gradle.kts new file mode 100644 index 0000000000..9144397b4f --- /dev/null +++ b/benchmarks/benchmark-commons/build.gradle.kts @@ -0,0 +1,15 @@ +plugins { + id("smithy-java.java-conventions") +} + +description = "Shared, non-published utilities for smithy-java benchmarks." + +dependencies { + compileOnly(libs.jmh.core) + testImplementation(libs.jmh.core) +} + +// Shared with the e2e runner, so this intentionally keeps the repository's +// Java 21 release target even though serde benchmarks execute on JDK 25. + +// Not published. No `smithy-java.module-conventions`, no publishing, no BOM entry. diff --git a/benchmarks/benchmark-commons/src/main/java/software/amazon/smithy/java/benchmarks/OpsPerCpuSecondProfiler.java b/benchmarks/benchmark-commons/src/main/java/software/amazon/smithy/java/benchmarks/OpsPerCpuSecondProfiler.java new file mode 100644 index 0000000000..2cc4f660e5 --- /dev/null +++ b/benchmarks/benchmark-commons/src/main/java/software/amazon/smithy/java/benchmarks/OpsPerCpuSecondProfiler.java @@ -0,0 +1,97 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.java.benchmarks; + +import java.util.Collection; +import java.util.List; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.IterationParams; +import org.openjdk.jmh.profile.InternalProfiler; +import org.openjdk.jmh.results.AggregationPolicy; +import org.openjdk.jmh.results.Aggregator; +import org.openjdk.jmh.results.IterationResult; +import org.openjdk.jmh.results.Result; +import org.openjdk.jmh.results.ResultRole; +import org.openjdk.jmh.util.SingletonStatistics; + +/** + * Reports completed benchmark operations per process CPU-second. + */ +public final class OpsPerCpuSecondProfiler implements InternalProfiler { + + public static final String METRIC_NAME = "ops_per_cpu_sec"; + + private long cpuTimeBefore; + + @Override + public String getDescription() { + return "Completed benchmark operations per process CPU-second"; + } + + @Override + public void beforeIteration(BenchmarkParams benchmarkParams, IterationParams iterationParams) { + cpuTimeBefore = ProcessCpuTime.now(); + } + + @Override + public Collection afterIteration( + BenchmarkParams benchmarkParams, + IterationParams iterationParams, + IterationResult result + ) { + long elapsedCpuNanos = ProcessCpuTime.now() - cpuTimeBefore; + long operations = result.getMetadata().getAllOps(); + return List.of(new OpsPerCpuSecondResult(operations, elapsedCpuNanos)); + } + + /** + * Aggregates the numerator and denominator independently so the final score + * is total operations divided by total CPU time, rather than an unweighted + * average of per-iteration ratios. + */ + static final class OpsPerCpuSecondResult extends Result { + + private static final long serialVersionUID = 1L; + + private final long operations; + private final long cpuNanos; + + OpsPerCpuSecondResult(long operations, long cpuNanos) { + super( + ResultRole.SECONDARY, + METRIC_NAME, + new SingletonStatistics(ProcessCpuTime.operationsPerSecond(operations, cpuNanos)), + "ops/CPU-sec", + AggregationPolicy.AVG); + this.operations = operations; + this.cpuNanos = cpuNanos; + } + + @Override + protected Aggregator getThreadAggregator() { + return new JoiningAggregator(); + } + + @Override + protected Aggregator getIterationAggregator() { + return new JoiningAggregator(); + } + + private static final class JoiningAggregator implements Aggregator { + + @Override + public OpsPerCpuSecondResult aggregate(Collection results) { + long operations = 0; + long cpuNanos = 0; + for (var result : results) { + operations += result.operations; + cpuNanos += result.cpuNanos; + } + return new OpsPerCpuSecondResult(operations, cpuNanos); + } + } + } +} diff --git a/benchmarks/benchmark-commons/src/main/java/software/amazon/smithy/java/benchmarks/ProcessCpuTime.java b/benchmarks/benchmark-commons/src/main/java/software/amazon/smithy/java/benchmarks/ProcessCpuTime.java new file mode 100644 index 0000000000..2a1f9366c9 --- /dev/null +++ b/benchmarks/benchmark-commons/src/main/java/software/amazon/smithy/java/benchmarks/ProcessCpuTime.java @@ -0,0 +1,59 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.java.benchmarks; + +import com.sun.management.OperatingSystemMXBean; +import java.lang.management.ManagementFactory; + +/** + * Utilities for measuring operations per process CPU-second. + */ +public final class ProcessCpuTime { + + private static final double NANOS_PER_SECOND = 1_000_000_000.0; + private static final OperatingSystemMXBean OS_MX_BEAN = loadOperatingSystemMxBean(); + + private ProcessCpuTime() {} + + /** + * Returns CPU time consumed by the current process, in nanoseconds. + * + * @return current process CPU time + * @throws IllegalStateException if process CPU time is unavailable + */ + public static long now() { + long cpuTime = OS_MX_BEAN.getProcessCpuTime(); + if (cpuTime < 0) { + throw new IllegalStateException("Process CPU time is not available on this JVM"); + } + return cpuTime; + } + + /** + * Computes completed operations per process CPU-second. + * + * @param operations number of completed operations + * @param elapsedCpuNanos process CPU time consumed by those operations + * @return completed operations per process CPU-second + */ + public static double operationsPerSecond(long operations, long elapsedCpuNanos) { + if (operations < 0) { + throw new IllegalArgumentException("operations must not be negative"); + } + if (elapsedCpuNanos <= 0) { + throw new IllegalArgumentException("elapsedCpuNanos must be positive"); + } + return operations * NANOS_PER_SECOND / elapsedCpuNanos; + } + + private static OperatingSystemMXBean loadOperatingSystemMxBean() { + var bean = ManagementFactory.getOperatingSystemMXBean(); + if (bean instanceof OperatingSystemMXBean osMxBean) { + return osMxBean; + } + throw new IllegalStateException("The current JVM does not expose process CPU time"); + } +} diff --git a/benchmarks/benchmark-commons/src/test/java/software/amazon/smithy/java/benchmarks/OpsPerCpuSecondProfilerTest.java b/benchmarks/benchmark-commons/src/test/java/software/amazon/smithy/java/benchmarks/OpsPerCpuSecondProfilerTest.java new file mode 100644 index 0000000000..b1f2ba8d19 --- /dev/null +++ b/benchmarks/benchmark-commons/src/test/java/software/amazon/smithy/java/benchmarks/OpsPerCpuSecondProfilerTest.java @@ -0,0 +1,24 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.java.benchmarks; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import org.junit.jupiter.api.Test; + +class OpsPerCpuSecondProfilerTest { + + @Test + void aggregatesOperationsAndCpuTimeIndependently() { + var first = new OpsPerCpuSecondProfiler.OpsPerCpuSecondResult(1_000, 1_000_000_000); + var second = new OpsPerCpuSecondProfiler.OpsPerCpuSecondResult(1_000, 3_000_000_000L); + + var aggregate = first.getIterationAggregator().aggregate(List.of(first, second)); + + assertThat(aggregate.getScore()).isEqualTo(500); + } +} diff --git a/benchmarks/benchmark-commons/src/test/java/software/amazon/smithy/java/benchmarks/ProcessCpuTimeTest.java b/benchmarks/benchmark-commons/src/test/java/software/amazon/smithy/java/benchmarks/ProcessCpuTimeTest.java new file mode 100644 index 0000000000..b1f196f69b --- /dev/null +++ b/benchmarks/benchmark-commons/src/test/java/software/amazon/smithy/java/benchmarks/ProcessCpuTimeTest.java @@ -0,0 +1,27 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.java.benchmarks; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; + +import org.junit.jupiter.api.Test; + +class ProcessCpuTimeTest { + + @Test + void computesOperationsPerCpuSecond() { + assertThat(ProcessCpuTime.operationsPerSecond(1_000, 500_000_000)).isEqualTo(2_000); + } + + @Test + void rejectsInvalidInputs() { + assertThatIllegalArgumentException() + .isThrownBy(() -> ProcessCpuTime.operationsPerSecond(-1, 1)); + assertThatIllegalArgumentException() + .isThrownBy(() -> ProcessCpuTime.operationsPerSecond(1, 0)); + } +} diff --git a/benchmarks/e2e-benchmarks/README.md b/benchmarks/e2e-benchmarks/README.md index 5e0630f62c..1cd713d3e7 100644 --- a/benchmarks/e2e-benchmarks/README.md +++ b/benchmarks/e2e-benchmarks/README.md @@ -2,6 +2,11 @@ A small fixed-workload runner that exercises the smithy-java SDK against live AWS services. +Each measurement reports `ops_per_cpu_sec` using process CPU time. Because +these benchmarks call live AWS services, their results are not comparable to +in-process serde benchmarks. Set `-De2e.collectMetrics=false` to exclude +resource-monitoring overhead. + ## Scope | Service | Operation | Variant | diff --git a/benchmarks/e2e-benchmarks/build.gradle.kts b/benchmarks/e2e-benchmarks/build.gradle.kts index ac628f437f..1f015ab183 100644 --- a/benchmarks/e2e-benchmarks/build.gradle.kts +++ b/benchmarks/e2e-benchmarks/build.gradle.kts @@ -13,6 +13,8 @@ application { } dependencies { + implementation(project(":benchmarks:benchmark-commons")) + // Codegen plugin and runtime modules referenced by generated client code. smithyBuild(project(":codegen:codegen-plugin")) smithyBuild(project(":client:client-core")) @@ -97,6 +99,16 @@ afterEvaluate { projectionPaths.forEach { srcDir("$it/resources") } } } + tasks.named("shadowJar") { + // Feed original service descriptors to Shadow. processResources writes + // duplicate paths to one output directory, which loses all but one + // projection before mergeServiceFiles() can see them. + projectionPaths.forEach { path -> + from("$path/resources") { + include("META-INF/services/**") + } + } + } } tasks.named("compileJava") { @@ -114,6 +126,10 @@ tasks.named("shadowJ archiveBaseName.set("smithy-java-e2e-benchmark-runner") archiveClassifier.set("") archiveVersion.set("") + duplicatesStrategy = DuplicatesStrategy.INCLUDE + filesNotMatching(listOf("META-INF/services/**", "META-INF/smithy/manifest")) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } mergeServiceFiles() // Keep META-INF/smithy/manifest entries from each codegen projection // so all schema indexes are discovered at runtime. diff --git a/benchmarks/e2e-benchmarks/src/main/java/software/amazon/smithy/java/benchmarks/e2e/BenchmarkSupport.java b/benchmarks/e2e-benchmarks/src/main/java/software/amazon/smithy/java/benchmarks/e2e/BenchmarkSupport.java index fcfb347ee5..ce80b58b80 100644 --- a/benchmarks/e2e-benchmarks/src/main/java/software/amazon/smithy/java/benchmarks/e2e/BenchmarkSupport.java +++ b/benchmarks/e2e-benchmarks/src/main/java/software/amazon/smithy/java/benchmarks/e2e/BenchmarkSupport.java @@ -12,6 +12,7 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import software.amazon.smithy.java.benchmarks.ProcessCpuTime; abstract class BenchmarkSupport { @@ -46,14 +47,18 @@ protected final void runMeasured(Action action) { } long startNs = System.nanoTime(); + long totalCpuDuration = 0; int lastSampleCount = 0; for (int i = 0; i < config.measurementBatches(); i++) { System.out.println("\nMeasurement batch " + (i + 1) + "/" + config.measurementBatches()); int operationsBefore = measuredCount.get(); long batchStart = System.nanoTime(); + long batchCpuStart = ProcessCpuTime.now(); executeBatch(pool, action, true); + long batchCpuDuration = ProcessCpuTime.now() - batchCpuStart; + totalCpuDuration += batchCpuDuration; long batchDuration = System.nanoTime() - batchStart; - printBatchResults(operationsBefore, batchDuration); + printBatchResults(operationsBefore, batchDuration, batchCpuDuration); if (config.collectMetrics()) { int now = monitor.sampleCount(); @@ -68,7 +73,7 @@ protected final void runMeasured(Action action) { } System.out.println("\n=== OVERALL RESULTS ==="); - printOverall(totalDuration); + printOverall(totalDuration, totalCpuDuration); } } @@ -138,7 +143,7 @@ private void printInit() { System.out.println(" Actions per batch: " + config.batchActions()); } - private void printBatchResults(int startIndex, long batchDurationNs) { + private void printBatchResults(int startIndex, long batchDurationNs, long batchCpuDurationNs) { int endIndex = measuredCount.get(); if (endIndex <= startIndex) { System.out.println(" No operations in this batch"); @@ -159,6 +164,7 @@ private void printBatchResults(int startIndex, long batchDurationNs) { count, batchSec, gbps); + printOpsPerCpuSecond(" ", count, batchCpuDurationNs); System.out.printf(" Latency (ms) - Avg: %.2f, P50: %.2f, P90: %.2f, P99: %.2f, Max: %.2f%n", avgNs / 1e6, p50 / 1e6, @@ -167,7 +173,7 @@ private void printBatchResults(int startIndex, long batchDurationNs) { max / 1e6); } - private void printOverall(long totalDurationNs) { + private void printOverall(long totalDurationNs, long totalCpuDurationNs) { int count = measuredCount.get(); if (count == 0) { System.out.println("No measurements collected"); @@ -186,6 +192,8 @@ private void printOverall(long totalDurationNs) { System.out.println("Total operations: " + count); System.out.printf("Total data transferred: %.2f MiB%n", totalBytes / 1024.0 / 1024.0); System.out.printf("Total duration: %.2f seconds%n", totalSec); + System.out.printf("Process CPU time: %.2f seconds%n", totalCpuDurationNs / 1e9); + printOpsPerCpuSecond("", count, totalCpuDurationNs); System.out.printf("Throughput: %.2f Gbps%n", gbps); System.out.println("\nLatency (milliseconds):"); System.out.printf(" Average: %.2f%n", avgNs / 1e6); @@ -204,6 +212,16 @@ private static double average(long[] values, int count) { return count == 0 ? 0 : (double) sum / count; } + private static void printOpsPerCpuSecond(String prefix, long operations, long cpuDurationNs) { + if (cpuDurationNs <= 0) { + System.out.println(prefix + "ops_per_cpu_sec: unavailable (CPU timer resolution too coarse)"); + } else { + System.out.printf("%sops_per_cpu_sec: %.2f%n", + prefix, + ProcessCpuTime.operationsPerSecond(operations, cpuDurationNs)); + } + } + @FunctionalInterface protected interface Action { default int prepare(int index) { diff --git a/benchmarks/serde-benchmarks/README.md b/benchmarks/serde-benchmarks/README.md index b6901aa22c..d753e4c178 100644 --- a/benchmarks/serde-benchmarks/README.md +++ b/benchmarks/serde-benchmarks/README.md @@ -39,7 +39,9 @@ happens once and is not measured. JMH is run in `SampleTime` mode so per-invocation latency samples are recorded; this gives the percentile distribution (p50/p90/p95/p99) needed by the shared -serde benchmark output schema. +serde benchmark output schema. A JMH profiler also measures process CPU time +and reports `ops_per_cpu_sec`. The score is total operations divided by total +process CPU time across the measurement iterations. ## Running @@ -101,7 +103,8 @@ The resulting JSON conforms to the shared schema: "p90": 960.0, "p95": 994.0, "p99": 1116.0, - "std_dev": 15.0 + "std_dev": 15.0, + "ops_per_cpu_sec": 1054852.0 } ] } diff --git a/benchmarks/serde-benchmarks/build.gradle.kts b/benchmarks/serde-benchmarks/build.gradle.kts index e3e46fcca4..1b4b93ecb3 100644 --- a/benchmarks/serde-benchmarks/build.gradle.kts +++ b/benchmarks/serde-benchmarks/build.gradle.kts @@ -33,6 +33,8 @@ dependencies { implementation(libs.smithy.protocol.test.traits) implementation(libs.smithy.utils) + jmhImplementation(project(":benchmarks:benchmark-commons")) + // The Smithy Java codegen plugin produces typed shape classes plus // ApiOperation classes per service (see `smithy-build.json`). The // client-core dep is required because the generated client classes @@ -137,6 +139,16 @@ afterEvaluate { srcDir(generateSmithyManifest) } } + tasks.jmhJar { + // Feed original service descriptors to Shadow. processJmhResources + // writes duplicate paths to one output directory, which loses all but + // one projection before mergeServiceFiles() can see them. + projectionPaths.forEach { path -> + from("$path/resources") { + include("META-INF/services/**") + } + } + } } tasks.named("processJmhResources") { @@ -159,6 +171,7 @@ tasks.named("compileJmhJava") { val fast = providers.gradleProperty("jmh.fast").isPresent jmh { benchmarkMode.set(listOf("sample")) + profilers.add("software.amazon.smithy.java.benchmarks.OpsPerCpuSecondProfiler") if (!fast) { warmupIterations = 5 iterations = 10 @@ -185,6 +198,10 @@ jmh { // mergeServiceFiles() so duplicate META-INF/services/ entries from // multiple codegen projections are concatenated rather than overwritten. tasks.jmhJar { + duplicatesStrategy = DuplicatesStrategy.INCLUDE + filesNotMatching(listOf("META-INF/services/**", "META-INF/smithy/manifest")) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } mergeServiceFiles() append("META-INF/smithy/manifest") } diff --git a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/JmhResultConverter.java b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/JmhResultConverter.java index 274a3961ef..70340eaa43 100644 --- a/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/JmhResultConverter.java +++ b/benchmarks/serde-benchmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/JmhResultConverter.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Locale; import java.util.stream.Collectors; +import software.amazon.smithy.java.benchmarks.OpsPerCpuSecondProfiler; import software.amazon.smithy.java.core.Version; import software.amazon.smithy.model.node.ArrayNode; import software.amazon.smithy.model.node.Node; @@ -58,7 +59,8 @@ * "p90": , * "p95": , * "p99": , - * "std_dev": + * "std_dev": , + * "ops_per_cpu_sec": * } * ] * } @@ -184,6 +186,17 @@ private static ArrayNode buildEntries(Node jmhResults) { var primary = result.getObjectMember("primaryMetric").orElse(Node.objectNode()); var percentiles = primary.getObjectMember("scorePercentiles").orElse(Node.objectNode()); + var opsPerCpuSec = result.getObjectMember("secondaryMetrics") + .flatMap(metrics -> metrics.getObjectMember(OpsPerCpuSecondProfiler.METRIC_NAME)) + .orElseThrow(() -> new IllegalArgumentException( + "Missing JMH secondary metric '" + OpsPerCpuSecondProfiler.METRIC_NAME + + "' for benchmark case '" + id + "'")); + double opsPerCpuSecScore = opsPerCpuSec.getNumberMember("score") + .map(NumberNode::getValue) + .map(Number::doubleValue) + .orElseThrow(() -> new IllegalArgumentException( + "JMH secondary metric '" + OpsPerCpuSecondProfiler.METRIC_NAME + + "' has no numeric score for benchmark case '" + id + "'")); entries.add(Node.objectNodeBuilder() .withMember("id", id) @@ -194,6 +207,7 @@ private static ArrayNode buildEntries(Node jmhResults) { .withMember("p95", doubleOf(percentiles, "95.0")) .withMember("p99", doubleOf(percentiles, "99.0")) .withMember("std_dev", doubleOf(primary, "scoreError")) + .withMember("ops_per_cpu_sec", opsPerCpuSecScore) .build()); } return ArrayNode.fromNodes(entries.toArray(new ObjectNode[0])); @@ -257,8 +271,8 @@ private static void writeMarkdown(ObjectNode output, Path file) throws IOExcepti pw.println("Args: " + jvm.getStringMemberOrDefault("args", "")); pw.println("```"); - pw.println("|id|n|mean|p50|p90|p95|p99|std_dev|"); - pw.println("|----:|----:|----:|----:|----:|----:|----:|----:|"); + pw.println("|id|n|mean|p50|p90|p95|p99|std_dev|ops_per_cpu_sec|"); + pw.println("|----:|----:|----:|----:|----:|----:|----:|----:|----:|"); for (Node bm : entries.getElements()) { if (!bm.isObjectNode()) { continue; @@ -272,6 +286,7 @@ private static void writeMarkdown(ObjectNode output, Path file) throws IOExcepti + "|" + nf.format(Math.round(doubleOf(entry, "p95"))) + "|" + nf.format(Math.round(doubleOf(entry, "p99"))) + "|" + nf.format(Math.round(doubleOf(entry, "std_dev"))) + + "|" + nf.format(Math.round(doubleOf(entry, "ops_per_cpu_sec"))) + "|"); } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b0761efd37..506fbaf134 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -43,6 +43,7 @@ smithy-jmespath = { module = "software.amazon.smithy:smithy-jmespath", version.r smithy-jmespath-tests = { module = "software.amazon.smithy:smithy-jmespath-tests", version.ref = "smithy" } smithy-waiters = { module = "software.amazon.smithy:smithy-waiters", version.ref = "smithy" } smithy-utils = { module = "software.amazon.smithy:smithy-utils", version.ref = "smithy" } +jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmhCore" } smithy-traitcodegen = { module = "software.amazon.smithy:smithy-trait-codegen", version.ref = "smithy" } smithy-rules = { module = "software.amazon.smithy:smithy-rules-engine", version.ref = "smithy" } smithy-aws-endpoints = { module = "software.amazon.smithy:smithy-aws-endpoints", version.ref = "smithy" } diff --git a/scripts/run-remote-benchmarks.sh b/scripts/run-remote-benchmarks.sh index 7ebbe4b03d..5bdc0a7d78 100755 --- a/scripts/run-remote-benchmarks.sh +++ b/scripts/run-remote-benchmarks.sh @@ -74,6 +74,7 @@ if [[ -z "$JAR" ]]; then exit 1 fi JAR_NAME=$(basename "$JAR") +REMOTE_CLASSPATH_DIR="$REMOTE_DIR/${JAR_NAME%.jar}-classpath" echo "==> Using jar: $JAR_NAME" # --- Copy jar to remote --- @@ -81,6 +82,13 @@ echo "==> Copying jar to $SSH_HOST:$REMOTE_DIR/" ssh "$SSH_HOST" "mkdir -p $REMOTE_DIR" scp -q "$JAR" "$SSH_HOST:$REMOTE_DIR/$JAR_NAME" +# The Gradle JMH plugin packages runtime dependencies as nested JARs and adds +# them to the manifest Class-Path. Extract the artifact on the remote host so +# those dependency JARs are available as real classpath entries. +echo "==> Extracting remote JMH classpath..." +ssh "$SSH_HOST" "rm -rf $REMOTE_CLASSPATH_DIR && mkdir -p $REMOTE_CLASSPATH_DIR \ + && cd $REMOTE_CLASSPATH_DIR && jar xf ../$JAR_NAME" + # --- Build JMH CLI args --- JVM_ARGS="-Xms1g -Xmx1g -XX:+UseG1GC -XX:+AlwaysPreTouch -Dsmithy-java.json-provider=smithy -Dsmithy-java.xml-provider=smithy" @@ -95,15 +103,19 @@ fi [[ -n "$INCLUDES" ]] && JMH_ARGS="$JMH_ARGS $INCLUDES" [[ -n "$TEST_CASE_ID" ]] && JMH_ARGS="$JMH_ARGS -p testCaseId=$TEST_CASE_ID" [[ -n "$PROFILERS" ]] && JMH_ARGS="$JMH_ARGS -prof $PROFILERS" +# Register this last so its measurement excludes other profilers' setup and +# teardown, matching the Gradle JMH configuration. +JMH_ARGS="$JMH_ARGS -prof software.amazon.smithy.java.benchmarks.OpsPerCpuSecondProfiler" # --- Run benchmarks on remote --- echo "==> Running benchmarks on $SSH_HOST..." -echo " $REMOTE_JAVA -jar $JAR_NAME $JMH_ARGS" -ssh "$SSH_HOST" "$REMOTE_JAVA -jar $REMOTE_DIR/$JAR_NAME $JMH_ARGS" +echo " $REMOTE_JAVA -cp '$REMOTE_CLASSPATH_DIR:$REMOTE_CLASSPATH_DIR/*' org.openjdk.jmh.Main $JMH_ARGS" +ssh "$SSH_HOST" "$REMOTE_JAVA -cp '$REMOTE_CLASSPATH_DIR:$REMOTE_CLASSPATH_DIR/*' \ + org.openjdk.jmh.Main $JMH_ARGS" # --- Run converter on remote --- echo "==> Converting results on $SSH_HOST..." -ssh "$SSH_HOST" "$REMOTE_JAVA -cp $REMOTE_DIR/$JAR_NAME \ +ssh "$SSH_HOST" "$REMOTE_JAVA -cp '$REMOTE_CLASSPATH_DIR:$REMOTE_CLASSPATH_DIR/*' \ software.amazon.smithy.java.benchmarks.serde.JmhResultConverter \ --input $REMOTE_DIR/results.json \ --output-prefix $REMOTE_DIR/output" @@ -120,4 +132,4 @@ echo "" echo "Results:" echo " JSON: $DEST/output.json" echo " Markdown: $DEST/output.md" -echo " Raw JMH: $DEST/results.json" \ No newline at end of file +echo " Raw JMH: $DEST/results.json" diff --git a/settings.gradle.kts b/settings.gradle.kts index 3c3677332f..d092ceff84 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -134,5 +134,6 @@ include(":model-bundle:model-bundle-api") // Benchmarks (not published) include(":benchmarks") +include(":benchmarks:benchmark-commons") include(":benchmarks:serde-benchmarks") include(":benchmarks:e2e-benchmarks")