Add ops/CPU-sec metric to benchmarks

benchmarks/benchmark-commons/build.gradle.kts

@@ -0,0 +1,15 @@
+plugins {
+    id("smithy-java.java-conventions")
+}
+
+description = "Shared, non-published utilities for smithy-java benchmarks."
+
+dependencies {
+    compileOnly(libs.jmh.core)
+    testImplementation(libs.jmh.core)
+}
+
+// Shared with the e2e runner, so this intentionally keeps the repository's
+// Java 21 release target even though serde benchmarks execute on JDK 25.
+
+// Not published. No `smithy-java.module-conventions`, no publishing, no BOM entry.

benchmarks/benchmark-commons/src/main/java/software/amazon/smithy/java/benchmarks/OpsPerCpuSecondProfiler.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.java.benchmarks;
+
+import java.util.Collection;
+import java.util.List;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.profile.InternalProfiler;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.results.Aggregator;
+import org.openjdk.jmh.results.IterationResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ResultRole;
+import org.openjdk.jmh.util.SingletonStatistics;
+
+/**
+ * Reports completed benchmark operations per process CPU-second.
+ */
+public final class OpsPerCpuSecondProfiler implements InternalProfiler {
+
+    public static final String METRIC_NAME = "ops_per_cpu_sec";
+
+    private long cpuTimeBefore;
+
+    @Override
+    public String getDescription() {
+        return "Completed benchmark operations per process CPU-second";
+    }
+
+    @Override
+    public void beforeIteration(BenchmarkParams benchmarkParams, IterationParams iterationParams) {
+        cpuTimeBefore = ProcessCpuTime.now();
+    }
+
+    @Override
+    public Collection<? extends Result> afterIteration(
+            BenchmarkParams benchmarkParams,
+            IterationParams iterationParams,
+            IterationResult result
+    ) {
+        long elapsedCpuNanos = ProcessCpuTime.now() - cpuTimeBefore;
+        long operations = result.getMetadata().getAllOps();
+        return List.of(new OpsPerCpuSecondResult(operations, elapsedCpuNanos));
+    }
+
+    /**
+     * Aggregates the numerator and denominator independently so the final score
+     * is total operations divided by total CPU time, rather than an unweighted
+     * average of per-iteration ratios.
+     */
+    static final class OpsPerCpuSecondResult extends Result<OpsPerCpuSecondResult> {
+
+        private static final long serialVersionUID = 1L;
+
+        private final long operations;
+        private final long cpuNanos;
+
+        OpsPerCpuSecondResult(long operations, long cpuNanos) {
+            super(
+                    ResultRole.SECONDARY,
+                    METRIC_NAME,
+                    new SingletonStatistics(ProcessCpuTime.operationsPerSecond(operations, cpuNanos)),
+                    "ops/CPU-sec",
+                    AggregationPolicy.AVG);
+            this.operations = operations;
+            this.cpuNanos = cpuNanos;
+        }
+
+        @Override
+        protected Aggregator<OpsPerCpuSecondResult> getThreadAggregator() {
+            return new JoiningAggregator();
+        }
+
+        @Override
+        protected Aggregator<OpsPerCpuSecondResult> getIterationAggregator() {
+            return new JoiningAggregator();
+        }
+
+        private static final class JoiningAggregator implements Aggregator<OpsPerCpuSecondResult> {
+
+            @Override
+            public OpsPerCpuSecondResult aggregate(Collection<OpsPerCpuSecondResult> results) {
+                long operations = 0;
+                long cpuNanos = 0;
+                for (var result : results) {
+                    operations += result.operations;
+                    cpuNanos += result.cpuNanos;
+                }
+                return new OpsPerCpuSecondResult(operations, cpuNanos);
+            }
+        }
+    }
+}

benchmarks/benchmark-commons/src/main/java/software/amazon/smithy/java/benchmarks/ProcessCpuTime.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.java.benchmarks;
+
+import com.sun.management.OperatingSystemMXBean;
+import java.lang.management.ManagementFactory;
+
+/**
+ * Utilities for measuring operations per process CPU-second.
+ */
+public final class ProcessCpuTime {
+
+    private static final double NANOS_PER_SECOND = 1_000_000_000.0;
+    private static final OperatingSystemMXBean OS_MX_BEAN = loadOperatingSystemMxBean();
+
+    private ProcessCpuTime() {}
+
+    /**
+     * Returns CPU time consumed by the current process, in nanoseconds.
+     *
+     * @return current process CPU time
+     * @throws IllegalStateException if process CPU time is unavailable
+     */
+    public static long now() {
+        long cpuTime = OS_MX_BEAN.getProcessCpuTime();
+        if (cpuTime < 0) {
+            throw new IllegalStateException("Process CPU time is not available on this JVM");
+        }
+        return cpuTime;
+    }
+
+    /**
+     * Computes completed operations per process CPU-second.
+     *
+     * @param operations number of completed operations
+     * @param elapsedCpuNanos process CPU time consumed by those operations
+     * @return completed operations per process CPU-second
+     */
+    public static double operationsPerSecond(long operations, long elapsedCpuNanos) {
+        if (operations < 0) {
+            throw new IllegalArgumentException("operations must not be negative");
+        }
+        if (elapsedCpuNanos <= 0) {
+            throw new IllegalArgumentException("elapsedCpuNanos must be positive");
+        }
+        return operations * NANOS_PER_SECOND / elapsedCpuNanos;
+    }
+
+    private static OperatingSystemMXBean loadOperatingSystemMxBean() {
+        var bean = ManagementFactory.getOperatingSystemMXBean();
+        if (bean instanceof OperatingSystemMXBean osMxBean) {
+            return osMxBean;
+        }
+        throw new IllegalStateException("The current JVM does not expose process CPU time");
+    }
+}

benchmarks/benchmark-commons/src/test/java/software/amazon/smithy/java/benchmarks/OpsPerCpuSecondProfilerTest.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.java.benchmarks;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.List;
+import org.junit.jupiter.api.Test;
+
+class OpsPerCpuSecondProfilerTest {
+
+    @Test
+    void aggregatesOperationsAndCpuTimeIndependently() {
+        var first = new OpsPerCpuSecondProfiler.OpsPerCpuSecondResult(1_000, 1_000_000_000);
+        var second = new OpsPerCpuSecondProfiler.OpsPerCpuSecondResult(1_000, 3_000_000_000L);
+
+        var aggregate = first.getIterationAggregator().aggregate(List.of(first, second));
+
+        assertThat(aggregate.getScore()).isEqualTo(500);
+    }
+}

benchmarks/benchmark-commons/src/test/java/software/amazon/smithy/java/benchmarks/ProcessCpuTimeTest.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.java.benchmarks;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException;
+
+import org.junit.jupiter.api.Test;
+
+class ProcessCpuTimeTest {
+
+    @Test
+    void computesOperationsPerCpuSecond() {
+        assertThat(ProcessCpuTime.operationsPerSecond(1_000, 500_000_000)).isEqualTo(2_000);
+    }
+
+    @Test
+    void rejectsInvalidInputs() {
+        assertThatIllegalArgumentException()
+                .isThrownBy(() -> ProcessCpuTime.operationsPerSecond(-1, 1));
+        assertThatIllegalArgumentException()
+                .isThrownBy(() -> ProcessCpuTime.operationsPerSecond(1, 0));
+    }
+}

benchmarks/e2e-benchmarks/README.md

@@ -2,6 +2,11 @@
 
 A small fixed-workload runner that exercises the smithy-java SDK against live AWS services.
 
+Each measurement reports `ops_per_cpu_sec` using process CPU time. Because
+these benchmarks call live AWS services, their results are not comparable to
+in-process serde benchmarks. Set `-De2e.collectMetrics=false` to exclude
+resource-monitoring overhead.
+
 ## Scope
 
 | Service  | Operation | Variant   |

benchmarks/e2e-benchmarks/build.gradle.kts

@@ -13,6 +13,8 @@ application {
 }
 
 dependencies {
+    implementation(project(":benchmarks:benchmark-commons"))
+
     // Codegen plugin and runtime modules referenced by generated client code.
     smithyBuild(project(":codegen:codegen-plugin"))
     smithyBuild(project(":client:client-core"))
@@ -97,6 +99,16 @@ afterEvaluate {
             projectionPaths.forEach { srcDir("$it/resources") }
         }
     }
+    tasks.named<com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar>("shadowJar") {
+        // Feed original service descriptors to Shadow. processResources writes
+        // duplicate paths to one output directory, which loses all but one
+        // projection before mergeServiceFiles() can see them.
+        projectionPaths.forEach { path ->
+            from("$path/resources") {
+                include("META-INF/services/**")
+            }
+        }
+    }
 }
 
 tasks.named("compileJava") {
@@ -114,6 +126,10 @@ tasks.named<com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar>("shadowJ
     archiveBaseName.set("smithy-java-e2e-benchmark-runner")
     archiveClassifier.set("")
     archiveVersion.set("")
+    duplicatesStrategy = DuplicatesStrategy.INCLUDE
+    filesNotMatching(listOf("META-INF/services/**", "META-INF/smithy/manifest")) {
+        duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+    }
     mergeServiceFiles()
     // Keep META-INF/smithy/manifest entries from each codegen projection
     // so all schema indexes are discovered at runtime.

benchmarks/e2e-benchmarks/src/main/java/software/amazon/smithy/java/benchmarks/e2e/BenchmarkSupport.java

@@ -12,6 +12,7 @@
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import software.amazon.smithy.java.benchmarks.ProcessCpuTime;
 
 abstract class BenchmarkSupport {
 
@@ -46,14 +47,18 @@ protected final void runMeasured(Action action) {
             }
 
             long startNs = System.nanoTime();
+            long totalCpuDuration = 0;
             int lastSampleCount = 0;
             for (int i = 0; i < config.measurementBatches(); i++) {
                 System.out.println("\nMeasurement batch " + (i + 1) + "/" + config.measurementBatches());
                 int operationsBefore = measuredCount.get();
                 long batchStart = System.nanoTime();
+                long batchCpuStart = ProcessCpuTime.now();
                 executeBatch(pool, action, true);
+                long batchCpuDuration = ProcessCpuTime.now() - batchCpuStart;
+                totalCpuDuration += batchCpuDuration;
                 long batchDuration = System.nanoTime() - batchStart;
-                printBatchResults(operationsBefore, batchDuration);
+                printBatchResults(operationsBefore, batchDuration, batchCpuDuration);
 
                 if (config.collectMetrics()) {
                     int now = monitor.sampleCount();
@@ -68,7 +73,7 @@ protected final void runMeasured(Action action) {
             }
 
             System.out.println("\n=== OVERALL RESULTS ===");
-            printOverall(totalDuration);
+            printOverall(totalDuration, totalCpuDuration);
         }
     }
 
@@ -138,7 +143,7 @@ private void printInit() {
         System.out.println("  Actions per batch: " + config.batchActions());
     }
 
-    private void printBatchResults(int startIndex, long batchDurationNs) {
+    private void printBatchResults(int startIndex, long batchDurationNs, long batchCpuDurationNs) {
         int endIndex = measuredCount.get();
         if (endIndex <= startIndex) {
             System.out.println("  No operations in this batch");
@@ -159,6 +164,7 @@ private void printBatchResults(int startIndex, long batchDurationNs) {
                 count,
                 batchSec,
                 gbps);
+        printOpsPerCpuSecond("  ", count, batchCpuDurationNs);
         System.out.printf("  Latency (ms) - Avg: %.2f, P50: %.2f, P90: %.2f, P99: %.2f, Max: %.2f%n",
                 avgNs / 1e6,
                 p50 / 1e6,
@@ -167,7 +173,7 @@ private void printBatchResults(int startIndex, long batchDurationNs) {
                 max / 1e6);
     }
 
-    private void printOverall(long totalDurationNs) {
+    private void printOverall(long totalDurationNs, long totalCpuDurationNs) {
         int count = measuredCount.get();
         if (count == 0) {
             System.out.println("No measurements collected");
@@ -186,6 +192,8 @@ private void printOverall(long totalDurationNs) {
         System.out.println("Total operations: " + count);
         System.out.printf("Total data transferred: %.2f MiB%n", totalBytes / 1024.0 / 1024.0);
         System.out.printf("Total duration: %.2f seconds%n", totalSec);
+        System.out.printf("Process CPU time: %.2f seconds%n", totalCpuDurationNs / 1e9);
+        printOpsPerCpuSecond("", count, totalCpuDurationNs);
         System.out.printf("Throughput: %.2f Gbps%n", gbps);
         System.out.println("\nLatency (milliseconds):");
         System.out.printf("  Average: %.2f%n", avgNs / 1e6);
@@ -204,6 +212,16 @@ private static double average(long[] values, int count) {
         return count == 0 ? 0 : (double) sum / count;
     }
 
+    private static void printOpsPerCpuSecond(String prefix, long operations, long cpuDurationNs) {
+        if (cpuDurationNs <= 0) {
+            System.out.println(prefix + "ops_per_cpu_sec: unavailable (CPU timer resolution too coarse)");
+        } else {
+            System.out.printf("%sops_per_cpu_sec: %.2f%n",
+                    prefix,
+                    ProcessCpuTime.operationsPerSecond(operations, cpuDurationNs));
+        }
+    }
+
     @FunctionalInterface
     protected interface Action {
         default int prepare(int index) {

benchmarks/serde-benchmarks/README.md

@@ -39,7 +39,9 @@ happens once and is not measured.
 
 JMH is run in `SampleTime` mode so per-invocation latency samples are recorded;
 this gives the percentile distribution (p50/p90/p95/p99) needed by the shared
-serde benchmark output schema.
+serde benchmark output schema. A JMH profiler also measures process CPU time
+and reports `ops_per_cpu_sec`. The score is total operations divided by total
+process CPU time across the measurement iterations.
 
 ## Running
 
@@ -101,7 +103,8 @@ The resulting JSON conforms to the shared schema:
       "p90": 960.0,
       "p95": 994.0,
       "p99": 1116.0,
-      "std_dev": 15.0
+      "std_dev": 15.0,
+      "ops_per_cpu_sec": 1054852.0
     }
   ]
 }