feat: Phase 2b — ingest and ETL layer

.gitguardian.yaml

@@ -1,30 +1,22 @@
-# GitGuardian configuration for this repository.
-# Schema reference: https://docs.gitguardian.com/ggshield-docs/reference/secret/ignore
-# (config file format defined by ggshield, the engine GitGuardian's scanning uses)
 version: 2
 
 secret:
-  # These two entries silence the "2 secrets uncovered" finding GitGuardian raised against the
-  # commit history introduced in PR #1 (feat/phase-1-render-kern) and inherited by every PR
-  # stacked on top of it (#2-#5).
-  #
-  # Why they are safe to ignore:
-  #   - Both values are login/password pairs for a MinIO container started by Testcontainers in
-  #     runner/src/test/java/net/onelitefeather/apus/runner/MinioFixtures.java. The container is
-  #     created and destroyed within a single integration test run; nothing outside that test
-  #     process ever talks to it.
-  #   - They were never used against any real MinIO deployment, staging or production system, or
-  #     any service reachable outside the throwaway test container.
-  #   - The root cause is already fixed: MinioFixtures now generates a fresh, random access
-  #     key/secret key on every test run instead of using these fixed literals (see that file's
-  #     ACCESS_KEY/SECRET_KEY fields). These two entries only silence the two now-historical
-  #     occurrences that remain in already-published commits on the stacked PR branches; rewriting
-  #     that history for two harmless test values would be disproportionate.
-  #
-  # Scoped to exactly these two literal values -- not a path or file exclusion -- so nothing else
-  # in this repository is exempted from scanning.
-  ignored_matches:
-    - name: MinIO test-container access key (runner integration tests, throwaway container)
-      match: apustest
-    - name: MinIO test-container secret key (runner integration tests, throwaway container)
-      match: apustestsecret
+  ignored-matches:
+    # Two MinIO credentials that the phase 1 integration tests used to hard-code.
+    # They were never real: both were handed to a throwaway Testcontainers MinIO that
+    # lives for the duration of one test run, and they existed nowhere else.
+    #
+    # The cause is fixed — every container test now generates its credentials per run
+    # from SecureRandom (see runner/src/test/.../MinioFixtures.java and the equivalents
+    # in the ingest and api modules). These entries exist only because the old literals
+    # remain in already-published commits, which cannot be rewritten here.
+    #
+    # Listed as SHA256 rather than plaintext, so this file does not itself contain the
+    # strings it exempts.
+    #
+    # Do not extend this list to silence new findings. A finding in new code means the
+    # code is wrong, not the scanner.
+    - name: "phase 1 MinIO test access key (throwaway container, cause fixed)"
+      match: bfd5d64da90af877034e91f582242391fea586e6d187a475a3407bf37bd6f422
+    - name: "phase 1 MinIO test secret key (throwaway container, cause fixed)"
+      match: 9c721c67b04a5ff4622f5796d44a042c328d5453795ac798e14c7a7a31125bdd

docs/superpowers/plans/2026-08-08-phase-1-render-kern.md

@@ -2218,8 +2218,8 @@ import org.testcontainers.utility.DockerImageName;
  */
 class RenderEndToEndTest {
 
-    private static final String ACCESS_KEY = "apustest";
-    private static final String SECRET_KEY = "apustestsecret";
+    private static final String ACCESS_KEY = "<generated-at-test-runtime>";
+    private static final String SECRET_KEY = "<generated-at-test-runtime>";
     private static final String WORLD_BUCKET = "bundles";
     private static final String MAP_BUCKET = "maps";
 
@@ -2399,8 +2399,8 @@ import org.testcontainers.utility.DockerImageName;
  */
 class TelemetryContractTest {
 
-    private static final String ACCESS_KEY = "apustest";
-    private static final String SECRET_KEY = "apustestsecret";
+    private static final String ACCESS_KEY = "<generated-at-test-runtime>";
+    private static final String SECRET_KEY = "<generated-at-test-runtime>";
 
     private static Path fixture() {
         return Path.of(System.getProperty("user.dir")).getParent().resolve("testdata/mini-world");

ingest/Dockerfile

@@ -0,0 +1,29 @@
+# syntax=docker/dockerfile:1
+
+FROM eclipse-temurin:25-jre-jammy
+
+# Non-root: the ingest job only ever writes below /work (its fetched source data lands in
+# /work/source; nothing else on the filesystem is touched). Same convention as runner/Dockerfile.
+RUN useradd --uid 10001 --create-home --home-dir /home/apus apus \
+ && mkdir -p /work/source \
+ && chown -R apus:apus /work
+
+COPY --chown=apus:apus ingest/entrypoint.sh /opt/apus/entrypoint.sh
+# Built by: ./gradlew :ingest:shadowJar
+COPY --chown=apus:apus ingest/build/libs/apus-ingest.jar /opt/apus/ingest.jar
+
+RUN chmod +x /opt/apus/entrypoint.sh
+
+USER apus
+WORKDIR /work
+
+# Only the genuinely optional variables get a default here. Every required variable
+# (APUS_SOURCE_TYPE, APUS_WORLD_NAME, APUS_SOURCE_VERSION, APUS_BUNDLE_*, APUS_S3_ENDPOINT/
+# ACCESS_KEY/SECRET_KEY, and the source-specific ones) intentionally has none, so a Job that
+# forgets one fails fast with IngestConfig's error message instead of silently doing the wrong
+# thing. See ingest/README.md for the full contract.
+ENV APUS_LAYOUT=auto \
+    APUS_S3_REGION=us-east-1 \
+    APUS_PROGRESS_INTERVAL_SECONDS=10
+
+ENTRYPOINT ["/opt/apus/entrypoint.sh"]

ingest/README.md

@@ -0,0 +1,157 @@
+# Apus Ingest Image
+
+Pulls raw Minecraft world data from a configured source (S3 bucket or a Pterodactyl panel
+backup), detects its on-disk layout, and writes it to S3 as a versioned, self-describing world
+bundle that the `runner` render container can consume without knowing where the data came from.
+
+`IngestMain` orchestrates the flow: read and validate configuration from environment variables
+(failing before anything is downloaded if one is missing) → fetch via the connector matching
+`APUS_SOURCE_TYPE` → detect the layout → write the bundle, manifest last.
+
+## Build
+
+```bash
+./gradlew :ingest:shadowJar
+docker build -f ingest/Dockerfile -t apus/ingest:dev .
+```
+
+The build context is the repository root, matching `runner/Dockerfile`'s convention -- the image
+needs the shadow jar Gradle builds under `ingest/build/libs/`.
+
+## Run
+
+```bash
+docker run --rm \
+  -e APUS_SOURCE_TYPE=s3 \
+  -e APUS_WORLD_NAME=world \
+  -e APUS_SOURCE_VERSION=2026-08-01T00-00-00Z.zip \
+  -e APUS_BUNDLE_BUCKET=bundles \
+  -e APUS_BUNDLE_TENANT=acme \
+  -e APUS_BUNDLE_SOURCE_NAME=survival-source \
+  -e APUS_BUNDLE_WORLD_ID=survival \
+  -e APUS_BUNDLE_VERSION=v1 \
+  -e APUS_S3_ENDPOINT=http://minio:9000 \
+  -e APUS_S3_ACCESS_KEY=... \
+  -e APUS_S3_SECRET_KEY=... \
+  -e APUS_SOURCE_S3_BUCKET=backups \
+  -e APUS_SOURCE_S3_PREFIX=survival/ \
+  -e APUS_SOURCE_S3_ACCESS_KEY=... \
+  -e APUS_SOURCE_S3_SECRET_KEY=... \
+  apus/ingest:dev
+```
+
+### Environment variables
+
+The full contract this image accepts, and the interface `IngestJobBuilder` (phase 2b, task 6)
+builds Kubernetes Jobs against -- the ingest equivalent of `runner/README.md`'s table.
+
+| Variable | Required | Default | Meaning |
+|---|---|---|---|
+| `APUS_SOURCE_TYPE` | yes | — | `s3` or `pterodactyl`. `upload`/`push` are recognised by `WorldSource.spec.type` but have no connector yet (phase 6) -- an unsupported value fails fast rather than being guessed at |
+| `APUS_WORLD_NAME` | yes | — | The world's folder name at the source, e.g. `world` |
+| `APUS_LAYOUT` | no | `auto` | `auto`, `vanilla`, or `bukkit`. `auto` lets `LayoutDetector` decide; any other value forces that layout and fails detection rather than falling back if the fetched data doesn't actually match it |
+| `APUS_SOURCE_VERSION` | yes | — | The exact source version id to fetch, as previously resolved by `WorldSourceReconciler`'s `discover()` poll (task 6) and recorded on the owning `WorldIngest.spec.sourceVersion`. This job never calls `discover()` itself -- see "Design notes" below |
+| `APUS_BUNDLE_BUCKET` | yes | — | Destination bucket for the bundle |
+| `APUS_BUNDLE_TENANT` | yes | — | Tenant id, becomes the first path segment of the bundle |
+| `APUS_BUNDLE_SOURCE_NAME` | yes | — | The owning `WorldSource`'s name, becomes the second path segment. Required so two different sources ingesting a world with the same id (e.g. the vanilla default `world`) never collide on the same bundle path -- see `BundlePath` |
+| `APUS_BUNDLE_WORLD_ID` | yes | — | World id, becomes the third path segment |
+| `APUS_BUNDLE_VERSION` | yes | — | This bundle version's identifier, becomes the fourth path segment |
+| `APUS_S3_ENDPOINT` | yes | — | Bundle destination S3-compatible endpoint, e.g. `http://minio:9000` |
+| `APUS_S3_ACCESS_KEY` | yes | — | Bundle destination access key |
+| `APUS_S3_SECRET_KEY` | yes | — | Bundle destination secret key |
+| `APUS_S3_REGION` | no | `us-east-1` | Bundle destination region |
+| `APUS_MC_VERSION` | no | — | Minecraft version recorded as `manifest.minecraftVersion`. Not part of the original task-5 contract -- added because nothing else can supply this value reliably; see "Design notes" |
+| `APUS_PROGRESS_INTERVAL_SECONDS` | no | `10` | Minimum seconds between progress lines on stdout; the final update always prints regardless |
+| `APUS_MAX_ARCHIVE_TOTAL_BYTES` | no | `5368709120` (5 GiB) | Upper bound on total bytes extracted from one source archive; extraction aborts once exceeded. The work directory has no mounted volume, so this bounds how much of the node's own disk an archive (hostile or just unexpectedly large) can consume -- see `Archives` |
+| `APUS_MAX_ARCHIVE_ENTRIES` | no | `200000` | Upper bound on the number of entries (files + directories) extracted from one source archive; extraction aborts once exceeded |
+| `APUS_SOURCE_S3_BUCKET` | yes, if `APUS_SOURCE_TYPE=s3` | — | Source bucket |
+| `APUS_SOURCE_S3_ENDPOINT` | no | AWS default | Source S3-compatible endpoint |
+| `APUS_SOURCE_S3_PREFIX` | no | `""` | Prefix under which each object is one fetchable version |
+| `APUS_SOURCE_S3_ACCESS_KEY` | no | credential chain | Source access key; if unset, falls back to the AWS SDK default credentials chain |
+| `APUS_SOURCE_S3_SECRET_KEY` | no | credential chain | Source secret key |
+| `APUS_SOURCE_S3_REGION` | no | `us-east-1` | Source region |
+| `APUS_PTERODACTYL_PANEL_URL` | yes, if `APUS_SOURCE_TYPE=pterodactyl` | — | Panel base URL, e.g. `https://panel.example.com` |
+| `APUS_PTERODACTYL_SERVER_ID` | yes, if pterodactyl | — | Server identifier (short id) |
+| `APUS_PTERODACTYL_API_KEY` | yes, if pterodactyl | — | Client API key (`ptlc_...`) |
+| `APUS_PTERODACTYL_WORLD_PATHS` | yes, if pterodactyl | — | Comma-separated top-level archive paths that make up the world, e.g. `world,world_nether,world_the_end` |
+
+Missing a required variable (including the source-specific ones for the chosen
+`APUS_SOURCE_TYPE`) aborts with a clear `[apus-ingest] ERROR: <VAR> is required but was not set.`
+message on stderr and a non-zero exit, **before** any connector is touched -- see
+`IngestConfig.fromEnv` and `IngestMainTest`.
+
+## Exit codes
+
+| Code | Meaning |
+|---|---|
+| `0` | Bundle written successfully |
+| `1` | Configuration error: a required variable is missing/blank, or `APUS_SOURCE_TYPE` names an unimplemented source. Nothing was fetched. |
+| `2` | Layout detection failed -- no known world layout (vanilla/bukkit) could be recognised in the fetched data. The error message names the paths that were actually found. |
+| `3` | Any other failure while fetching the source or writing the bundle (network error, S3 error, ...) |
+
+## Design notes
+
+**Progress reporting: stdout lines, no HTTP server.** Unlike `runner`, which stays alive for
+minutes serving `/progress` to an operator that polls it during a long render, the ingest job is
+short-lived and its Kubernetes `Job`/`Pod` status already gives an external reconciler
+(`WorldIngestReconciler`, task 6) the coarse state it needs -- `Active`/`Succeeded`/`Failed` plus
+timestamps, with no extra moving part in the container. Building an HTTP server here would add a
+listening port, a shutdown-ordering concern, and a second thing that can fail, for a job that
+typically finishes before a poll loop would notice it existed. Instead, `IngestMain` prints a
+`phase=<Pending|Extracting|Transforming|Loading|Succeeded|Failed>` line at each stage transition
+and `ThrottledProgressSink` prints a `progress: NN.N% (done/total bytes)` line at most once per
+`APUS_PROGRESS_INTERVAL_SECONDS` (plus unconditionally on the final update) while the bundle is
+being written -- exactly the periodic-line-plus-end-state shape the phase 2b plan asks for. If a
+future reconciler wants finer-grained percentage rather than just phase, it already has one: read
+the pod's logs and parse this same line format, the same relationship `TelemetryContractTest`
+documents between `runner`'s log-tail route and its `/progress` endpoint, just without needing an
+HTTP round trip at all here.
+
+**Minecraft version: environment variable, not parsed from `level.dat`.** The Minecraft version a
+world was generated/played under lives in `level.dat`'s NBT `Data.Version.Name` (or, on very old
+worlds, isn't present at all and must be inferred from `Data.version`, an integer data-version
+with its own separate mapping table). Reading it reliably means either adding an NBT-parsing
+dependency (none of the ones already in this project's catalog expose it publicly at the ingest
+module's layer -- `bluemap-core` has one internally, but `ingest` deliberately does not depend on
+BlueMap) or hand-rolling a gzip+NBT reader for a single, easily-gotten-wrong field, for every
+supported Minecraft version's `level.dat` shape. Given that `WorldSource`/`WorldIngest` are
+already tenant-authored custom resources, this was decided against in favour of a user-supplied
+field: `WorldSource.spec.worlds[].minecraftVersion`, which `IngestJobBuilder` reads for the
+matching world selector and passes straight through as `APUS_MC_VERSION` -- the tenant already
+knows which version they run, and a wrong guess parsed out of `level.dat` would silently mislabel
+a manifest forever instead. `APUS_MC_VERSION` itself stays optional at this image's own contract
+level (nothing here requires the operator to have set it), and if unset,
+`manifest.minecraftVersion` stays `null`, matching `BundleManifest`'s existing "or `null` if not
+known at bundle time" contract rather than inventing a new failure mode. If a future task adds
+real NBT parsing, it becomes an *additional* fallback ahead of the environment variable, not a
+replacement for it -- `level.dat` is genuinely absent for connectors that only fetch specific
+region files, so the variable stays useful either way.
+
+**Source version, not `discover()`, inside the job.** `WorldSourceConnector.discover()` is a
+polling operation -- it belongs to `WorldSourceReconciler` (task 6), which resolves "is there a
+new version" on a schedule and records the chosen id on `WorldIngest.spec.sourceVersion`. The job
+itself only ever calls `fetch()` for the one version it was told to fetch (`APUS_SOURCE_VERSION`);
+it never lists what's available. This keeps a single ingest run deterministic and keeps the
+"what's new" decision in exactly one place.
+
+## Integration tests
+
+`S3SourceConnectorTest` (`ingest/src/test/java/net/onelitefeather/apus/ingest/connector/`) starts
+a real MinIO container via Testcontainers and therefore needs Docker. Like `runner` and
+`operator` do for their own container-based tests, it is **not** part of `./gradlew build` or
+`check` -- it is excluded from the default `test` task and runs only via the explicit task below:
+
+```bash
+./gradlew :ingest:integrationTest
+```
+
+Every other test in this module (`IngestConfigTest`, `IngestMainTest`, `ThrottledProgressSinkTest`,
+`BundleManifestTest`, `BundleWriterTest`, `LayoutDetectorTest`, `S3ClientTest`,
+`PterodactylConnectorTest`, `ArchivesTest`, `TarStreamReaderTest`) runs Docker-free as part of the
+routine `./gradlew :ingest:test`.
+
+A full source-to-bundle-to-render end-to-end test (ingest a Bukkit-layout world fixture against
+real MinIO, check the resulting manifest, then start a real render against the produced bundle
+with the `runner` image) lives in `runner`'s `:runner:integrationTest`
+(`IngestRenderContractTest`), not here -- proving the contract between this module's output and
+`runner`'s input needs both modules in the same test.

ingest/build.gradle.kts

@@ -0,0 +1,65 @@
+import java.time.Duration
+
+plugins {
+    application
+    alias(libs.plugins.shadow)
+}
+
+dependencies {
+    // AWS SDK v2 S3 client -- see settings.gradle.kts for why this over the MinIO Java client.
+    implementation(platform(libs.aws.sdk.bom))
+    implementation(libs.aws.sdk.s3)
+
+    // Jackson -- see settings.gradle.kts for why. Backs BundleManifest (de)serialisation and
+    // Pterodactyl API response parsing; both used to be hand-rolled JSON codecs.
+    implementation(platform(libs.jackson.bom))
+    implementation(libs.jackson.databind)
+
+    testImplementation(platform(libs.junit.bom))
+    testImplementation(libs.junit.jupiter)
+    testRuntimeOnly(libs.junit.platform.launcher)
+
+    // Real MinIO via Testcontainers for S3SourceConnectorTest -- see that test's Javadoc.
+    testImplementation(platform(libs.testcontainers.bom))
+    testImplementation(libs.testcontainers.junit)
+    testImplementation(libs.testcontainers.minio)
+}
+
+application {
+    mainClass.set("net.onelitefeather.apus.ingest.IngestMain")
+}
+
+tasks {
+    shadowJar {
+        archiveClassifier.set("")
+        archiveBaseName.set("apus-ingest")
+        // Fixed name instead of the default "apus-ingest-<version>.jar": ingest/Dockerfile
+        // COPYs this file by name (no glob) -- see telemetry-addon/build.gradle.kts for the
+        // same rationale applied to the render container's addon jar.
+        archiveFileName.set("apus-ingest.jar")
+    }
+    build {
+        dependsOn(shadowJar)
+    }
+}
+
+// S3SourceConnectorTest starts a real MinIO container via Testcontainers and therefore needs
+// Docker. Exactly like runner/build.gradle.kts and operator/build.gradle.kts do for their own
+// container-based tests, that must not run as part of the routine `./gradlew build`/`check` --
+// it would make every build slow and fail outright on a machine without Docker. Excluded from
+// the default `test` task and exposed only via the explicit `integrationTest` task below. See
+// ingest/README.md for how to run it.
+tasks.test {
+    exclude("**/S3SourceConnectorTest.class")
+}
+
+val integrationTest by tasks.registering(Test::class) {
+    group = "verification"
+    description = "Runs S3SourceConnectorTest against a real MinIO container via Testcontainers. " +
+        "Requires Docker. Not part of build/check."
+    testClassesDirs = sourceSets.test.get().output.classesDirs
+    classpath = sourceSets.test.get().runtimeClasspath
+    include("**/S3SourceConnectorTest.class")
+    timeout.set(Duration.ofMinutes(5))
+    outputs.upToDateWhen { false }
+}

ingest/entrypoint.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Required configuration is validated by IngestMain itself, before it does any work (see
+# IngestConfig.fromEnv) -- this entrypoint stays a thin, replaceable launch line, the same
+# separation of concerns runner/entrypoint.sh uses between shell-level checks and the actual
+# render process.
+#
+# No secret ever appears here: JAVA_OPTS carries only JVM tuning flags, and every credential
+# IngestMain needs is read straight out of its own environment in-process, never passed as a
+# command-line argument to this or any other process (unlike `mc alias set` in
+# runner/bin/bundle-sync.sh, which is why that script writes a config file instead).
+echo "[apus-ingest] starting"
+# shellcheck disable=SC2086
+exec java ${JAVA_OPTS:-} -jar /opt/apus/ingest.jar