From 3294d43a6999fb590330a1f554473dae18d89cb9 Mon Sep 17 00:00:00 2001 From: TheMeinerLP Date: Sun, 9 Aug 2026 11:58:45 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20Phase=202b=20=E2=80=94=20ingest=20and?= =?UTF-8?q?=20ETL=20layer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ingest/ETL layer on top of Phase 2a. A WorldSource is now configured once and Apus takes it from there: it polls for new world data, normalises whatever layout it finds into a versioned World Bundle in S3, and the render path picks it up without knowing where the world came from. - LayoutDetector recognises vanilla, Bukkit and nested layouts, normalising all of them to overworld/the_nether/the_end; BundleWriter writes the bundle with the manifest last so half-unpacked worlds are impossible without S3 transactions; S3SourceConnector and PterodactylConnector are the only source-specific pieces; WorldSource/WorldIngest CRDs and reconcilers handle cron-driven polling, job orchestration and retention; the ingest container image runs the ETL as a Kubernetes Job. - IngestRenderContractTest proves the contract end to end: real ingest against a Bukkit-layout world in real MinIO, rendered by the Phase 1 runner image, with no production code changes required to make it hold. - Fixes several security findings in this untrusted-input layer: LayoutDetector followed symlinks and did not confine paths to the work root; archive extraction had no size limits (zip-bomb risk); signed Pterodactyl download URLs leaked into logs; a WorldIngest could drive a foreign WorldSource because only the name was checked, not UID ownership. - Fixes two correctness bugs: an ingest could report success while its bundle stayed invisible (terminality now comes only from Job status, not a log line race), and retention could delete another source's live bundle when two sources shared a world name (bundle paths are now scoped by source). - 160 operator tests, 65 ingest tests, plus integration tests against real MinIO and k3s. This branch replaces clean/phase-2b-ingest (PR #11), stacked on clean2/phase-2-operator instead of clean/phase-2-operator. Same content otherwise. --- .../plans/2026-08-08-phase-2b-ingest.md | 323 ++++++++ ingest/Dockerfile | 29 + ingest/README.md | 157 ++++ ingest/build.gradle.kts | 65 ++ ingest/entrypoint.sh | 15 + .../apus/ingest/BundleManifest.java | 125 ++++ .../apus/ingest/BundlePath.java | 59 ++ .../apus/ingest/BundleWriter.java | 322 ++++++++ .../apus/ingest/IngestConfig.java | 384 ++++++++++ .../apus/ingest/IngestMain.java | 164 ++++ .../apus/ingest/LayoutDetector.java | 274 +++++++ .../onelitefeather/apus/ingest/S3Client.java | 43 ++ .../apus/ingest/ThrottledProgressSink.java | 68 ++ .../apus/ingest/WorldLayout.java | 42 ++ .../apus/ingest/connector/Archives.java | 258 +++++++ .../connector/PterodactylConnector.java | 287 +++++++ .../ingest/connector/S3SourceConnector.java | 164 ++++ .../apus/ingest/connector/SourceVersion.java | 34 + .../ingest/connector/TarStreamReader.java | 254 +++++++ .../connector/WorldSourceConnector.java | 57 ++ .../apus/ingest/BundleManifestTest.java | 148 ++++ .../apus/ingest/BundleWriterTest.java | 402 ++++++++++ .../apus/ingest/IngestConfigTest.java | 226 ++++++ .../apus/ingest/IngestMainTest.java | 73 ++ .../apus/ingest/LayoutDetectorTest.java | 174 +++++ .../apus/ingest/S3ClientTest.java | 70 ++ .../ingest/ThrottledProgressSinkTest.java | 94 +++ .../apus/ingest/connector/ArchivesTest.java | 134 ++++ .../connector/PterodactylConnectorTest.java | 323 ++++++++ .../connector/S3SourceConnectorTest.java | 199 +++++ .../ingest/connector/TarStreamReaderTest.java | 145 ++++ .../apus/ingest/connector/TestTarBuilder.java | 173 +++++ operator/build.gradle.kts | 19 + .../apus/operator/ApusOperator.java | 11 +- .../apus/operator/OperatorConfig.java | 58 +- .../apus/operator/api/BundleRef.java | 60 ++ .../apus/operator/api/Labels.java | 15 + .../apus/operator/api/WorldIngest.java | 49 ++ .../apus/operator/api/WorldIngestSpec.java | 55 ++ .../apus/operator/api/WorldIngestStatus.java | 126 ++++ .../apus/operator/api/WorldSource.java | 99 +++ .../apus/operator/api/WorldSourceSpec.java | 185 +++++ .../apus/operator/api/WorldSourceStatus.java | 106 +++ .../apus/operator/ingest/AwsBundleStore.java | 111 +++ .../apus/operator/ingest/BundleStore.java | 48 ++ .../apus/operator/ingest/CronSchedule.java | 100 +++ .../operator/ingest/IngestJobBuilder.java | 383 ++++++++++ .../operator/ingest/IngestLogProgress.java | 86 +++ .../apus/operator/ingest/Secrets.java | 59 ++ .../ingest/WorldIngestReconciler.java | 625 ++++++++++++++++ .../ingest/WorldSourceReconciler.java | 396 ++++++++++ .../render/BlueMapRenderReconciler.java | 14 +- .../apus/operator/ApusOperatorTest.java | 8 +- .../apus/operator/CrdGenerationTest.java | 33 + .../apus/operator/OperatorConfigTest.java | 26 +- .../apus/operator/api/IngestResourceTest.java | 116 +++ .../operator/ingest/CronScheduleTest.java | 81 ++ .../operator/ingest/IngestJobBuilderTest.java | 255 +++++++ .../ingest/IngestLogProgressTest.java | 89 +++ .../ingest/WorldIngestReconcilerTest.java | 698 ++++++++++++++++++ .../ingest/WorldSourceReconcilerTest.java | 312 ++++++++ .../render/BlueMapRenderReconcilerTest.java | 22 + .../operator/render/RenderJobBuilderTest.java | 11 +- runner/build.gradle.kts | 7 + .../apus/runner/IngestRenderContractTest.java | 375 ++++++++++ settings.gradle.kts | 43 +- 66 files changed, 9944 insertions(+), 22 deletions(-) create mode 100644 docs/superpowers/plans/2026-08-08-phase-2b-ingest.md create mode 100644 ingest/Dockerfile create mode 100644 ingest/README.md create mode 100644 ingest/build.gradle.kts create mode 100644 ingest/entrypoint.sh create mode 100644 ingest/src/main/java/net/onelitefeather/apus/ingest/BundleManifest.java create mode 100644 ingest/src/main/java/net/onelitefeather/apus/ingest/BundlePath.java create mode 100644 ingest/src/main/java/net/onelitefeather/apus/ingest/BundleWriter.java create mode 100644 ingest/src/main/java/net/onelitefeather/apus/ingest/IngestConfig.java create mode 100644 ingest/src/main/java/net/onelitefeather/apus/ingest/IngestMain.java create mode 100644 ingest/src/main/java/net/onelitefeather/apus/ingest/LayoutDetector.java create mode 100644 ingest/src/main/java/net/onelitefeather/apus/ingest/S3Client.java create mode 100644 ingest/src/main/java/net/onelitefeather/apus/ingest/ThrottledProgressSink.java create mode 100644 ingest/src/main/java/net/onelitefeather/apus/ingest/WorldLayout.java create mode 100644 ingest/src/main/java/net/onelitefeather/apus/ingest/connector/Archives.java create mode 100644 ingest/src/main/java/net/onelitefeather/apus/ingest/connector/PterodactylConnector.java create mode 100644 ingest/src/main/java/net/onelitefeather/apus/ingest/connector/S3SourceConnector.java create mode 100644 ingest/src/main/java/net/onelitefeather/apus/ingest/connector/SourceVersion.java create mode 100644 ingest/src/main/java/net/onelitefeather/apus/ingest/connector/TarStreamReader.java create mode 100644 ingest/src/main/java/net/onelitefeather/apus/ingest/connector/WorldSourceConnector.java create mode 100644 ingest/src/test/java/net/onelitefeather/apus/ingest/BundleManifestTest.java create mode 100644 ingest/src/test/java/net/onelitefeather/apus/ingest/BundleWriterTest.java create mode 100644 ingest/src/test/java/net/onelitefeather/apus/ingest/IngestConfigTest.java create mode 100644 ingest/src/test/java/net/onelitefeather/apus/ingest/IngestMainTest.java create mode 100644 ingest/src/test/java/net/onelitefeather/apus/ingest/LayoutDetectorTest.java create mode 100644 ingest/src/test/java/net/onelitefeather/apus/ingest/S3ClientTest.java create mode 100644 ingest/src/test/java/net/onelitefeather/apus/ingest/ThrottledProgressSinkTest.java create mode 100644 ingest/src/test/java/net/onelitefeather/apus/ingest/connector/ArchivesTest.java create mode 100644 ingest/src/test/java/net/onelitefeather/apus/ingest/connector/PterodactylConnectorTest.java create mode 100644 ingest/src/test/java/net/onelitefeather/apus/ingest/connector/S3SourceConnectorTest.java create mode 100644 ingest/src/test/java/net/onelitefeather/apus/ingest/connector/TarStreamReaderTest.java create mode 100644 ingest/src/test/java/net/onelitefeather/apus/ingest/connector/TestTarBuilder.java create mode 100644 operator/src/main/java/net/onelitefeather/apus/operator/api/BundleRef.java create mode 100644 operator/src/main/java/net/onelitefeather/apus/operator/api/WorldIngest.java create mode 100644 operator/src/main/java/net/onelitefeather/apus/operator/api/WorldIngestSpec.java create mode 100644 operator/src/main/java/net/onelitefeather/apus/operator/api/WorldIngestStatus.java create mode 100644 operator/src/main/java/net/onelitefeather/apus/operator/api/WorldSource.java create mode 100644 operator/src/main/java/net/onelitefeather/apus/operator/api/WorldSourceSpec.java create mode 100644 operator/src/main/java/net/onelitefeather/apus/operator/api/WorldSourceStatus.java create mode 100644 operator/src/main/java/net/onelitefeather/apus/operator/ingest/AwsBundleStore.java create mode 100644 operator/src/main/java/net/onelitefeather/apus/operator/ingest/BundleStore.java create mode 100644 operator/src/main/java/net/onelitefeather/apus/operator/ingest/CronSchedule.java create mode 100644 operator/src/main/java/net/onelitefeather/apus/operator/ingest/IngestJobBuilder.java create mode 100644 operator/src/main/java/net/onelitefeather/apus/operator/ingest/IngestLogProgress.java create mode 100644 operator/src/main/java/net/onelitefeather/apus/operator/ingest/Secrets.java create mode 100644 operator/src/main/java/net/onelitefeather/apus/operator/ingest/WorldIngestReconciler.java create mode 100644 operator/src/main/java/net/onelitefeather/apus/operator/ingest/WorldSourceReconciler.java create mode 100644 operator/src/test/java/net/onelitefeather/apus/operator/api/IngestResourceTest.java create mode 100644 operator/src/test/java/net/onelitefeather/apus/operator/ingest/CronScheduleTest.java create mode 100644 operator/src/test/java/net/onelitefeather/apus/operator/ingest/IngestJobBuilderTest.java create mode 100644 operator/src/test/java/net/onelitefeather/apus/operator/ingest/IngestLogProgressTest.java create mode 100644 operator/src/test/java/net/onelitefeather/apus/operator/ingest/WorldIngestReconcilerTest.java create mode 100644 operator/src/test/java/net/onelitefeather/apus/operator/ingest/WorldSourceReconcilerTest.java create mode 100644 runner/src/test/java/net/onelitefeather/apus/runner/IngestRenderContractTest.java diff --git a/docs/superpowers/plans/2026-08-08-phase-2b-ingest.md b/docs/superpowers/plans/2026-08-08-phase-2b-ingest.md new file mode 100644 index 0000000..d814b68 --- /dev/null +++ b/docs/superpowers/plans/2026-08-08-phase-2b-ingest.md @@ -0,0 +1,323 @@ +# Apus Phase 2b — Ingest und ETL: Implementierungsplan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Welt-Daten aus heterogenen Quellen in ein einheitliches, versioniertes World Bundle in S3 überführen, sodass der Render-Pfad aus Phase 2a sie ohne Kenntnis der Herkunft verarbeiten kann. + +**Architecture:** Nur der Extract-Schritt ist quellenspezifisch; Transform (Layout-Erkennung) und Load (Bundle-Writer) sind gemeinsam. Ein neuer Connector kostet damit eine Implementierung von zwei Methoden. Zwei neue Custom Resources (`WorldSource`, `WorldIngest`) reihen sich in das Muster aus Phase 2a ein: Vorlage erzeugt Ausführungen. Der eigentliche Ingest läuft als Kubernetes-Job mit einem eigenen Container-Image, analog zum Runner aus Phase 1. + +**Tech Stack:** Java 25, Gradle, JOSDK 5.5.1, Fabric8 7.8.0, JUnit Jupiter, Testcontainers (MinIO), Fabric8 Mock-Server. + +## Global Constraints + +- **Java-Toolchain 25**, Basispaket `net.onelitefeather.apus.ingest` (neues Modul `ingest`) bzw. `net.onelitefeather.apus.operator.api` für die CRDs. +- API-Gruppe `bluemap.onelitefeather.net`, Version `v1alpha1`. Beide neuen Ressourcen sind **namespaced**. +- Koordinaten wie in Phase 2a; der Fabric8-Client kommt transitiv über JOSDK. +- **Das Bundle-Manifest ist der Vertrag** (§5 der Spec). Es wird **zuletzt** geschrieben — es ist der Commit-Punkt. Ohne Manifest existiert ein Bundle nicht und wird nie gerendert. Damit gibt es keine halb entpackten Welten im Render-Pfad. +- **Bundles sind unveränderlich.** Neue Welt-Daten erzeugen eine neue Version, nie eine Änderung an einer bestehenden. +- **Die Regionsliste gehört ins Manifest.** Sie kostet beim Ingest nichts, weil ohnehin jede `.mca`-Datei angefasst wird, und ist die Grundlage für das Sharding aus Phase 4 sowie für genaue Fortschrittsberechnung. +- **Dimensionen werden logisch benannt** (`overworld`, `the_nether`, `the_end`), unabhängig davon, ob die Quelle Vanilla- oder Bukkit-Layout hatte. +- **Eigentümerprüfung**: Vor dem Verändern einer bestehenden Ressource ist zu prüfen, ob sie zur eigenen Custom Resource gehört (Name **und** UID). Fremde oder ungekennzeichnete Ressourcen führen zu einer Konflikt-Condition. Das war ein Sicherheitsbefund in Phase 2a und darf sich nicht wiederholen. +- **Gemeinsame `Labels`-Klasse** aus Phase 2a für alle erzeugten Ressourcen. +- Zugangsdaten niemals in Status, Events oder Logs. +- AGPL-Header über Spotless, Conventional Commits, **keine** Claude-Attribution, Bezeichner und Javadoc auf Englisch. + +### Was aus Phase 1 und 2a bereits existiert und zu benutzen ist + +- `net.onelitefeather.apus.operator.api.Labels`, `Conditions`, `Ref`, `OperatorConfig` +- Das Eigentümer-Prüfmuster in `TenantReconciler` und `BlueMapMapReconciler` +- `client.supports(...)` als Prüfung auf fehlende fremde CRDs +- Die CRD-Generierung erfasst neue Ressourcen unter `net.onelitefeather.apus.operator.api` automatisch +- `BlueMapRender.spec.bundleUrl` erwartet eine `s3://`-URL auf ein Bundle-Verzeichnis + +--- + +## File Structure + +``` +ingest/ neues Modul, Container-Image analog zu runner/ +├── build.gradle.kts +├── Dockerfile +└── src/ + ├── main/java/net/onelitefeather/apus/ingest/ + │ ├── IngestMain.java Einstiegspunkt des Jobs + │ ├── WorldLayout.java Erkanntes Layout + Dimensions-Zuordnung + │ ├── LayoutDetector.java Erkennt vanilla / bukkit / nested + │ ├── BundleManifest.java Datenmodell des Manifests + │ ├── BundleWriter.java Schreibt Bundle nach S3, Manifest zuletzt + │ ├── S3Client.java Schmale S3-Fassade (Upload, List, Head) + │ └── connector/ + │ ├── WorldSourceConnector.java Schnittstelle: discover / fetch + │ ├── SourceVersion.java + │ ├── S3SourceConnector.java Pull aus einem Bucket-Prefix + │ └── PterodactylConnector.java Pull über die Panel-API + └── test/java/... + +operator/src/main/java/net/onelitefeather/apus/operator/ +├── api/WorldSource.java WorldSourceSpec.java WorldSourceStatus.java +├── api/WorldIngest.java WorldIngestSpec.java WorldIngestStatus.java +└── ingest/ + ├── WorldSourceReconciler.java Poll-Zeitplan → erzeugt WorldIngest + ├── WorldIngestReconciler.java erzeugt den Ingest-Job, führt Fortschritt + └── IngestJobBuilder.java baut den Job aus dem ingest-Image +``` + +**Warum ein eigenes Modul:** Der Ingest läuft als Job im Cluster, nicht im Operator-Prozess. Ein großes `tar.gz` zu streamen und Gigabyte an Region-Dateien zu schreiben gehört nicht in einen Operator, der viele Ressourcen gleichzeitig betreut. `LayoutDetector`, `BundleManifest` und die Connectoren sind reine Logik und ohne Cluster testbar. + +--- + +## Parallelisierung + +Dasselbe Muster wie in Phase 2a: Datenmodell zuerst, dann berühren die Folgeaufgaben getrennte Dateien. + +| Gruppe | Aufgaben | Ausführung | +|---|---|---| +| A | Task 1 — Modul und Datenmodell | sequenziell | +| B | Task 2, Task 3, Task 4 | **parallel**, je eigener Worktree | +| C | Task 5 — Ingest-Einstiegspunkt und Image | sequenziell | +| D | Task 6 — Reconciler | sequenziell | +| E | Task 7 — Integrationstest | sequenziell | + +**Dateien der parallelen Gruppe** (disjunkt): +- Task 2: `LayoutDetector.java`, `WorldLayout.java` + Tests +- Task 3: `BundleManifest.java`, `BundleWriter.java`, `S3Client.java` + Tests +- Task 4: `connector/*` + Tests + +--- + +### Task 1: Modul, CRDs und gemeinsames Datenmodell + +**Files:** +- Modify: `settings.gradle.kts` (Modul `ingest`, Katalog-Einträge für den S3-Client) +- Create: `ingest/build.gradle.kts` +- Create: `operator/src/main/java/.../api/WorldSource.java`, `WorldSourceSpec.java`, `WorldSourceStatus.java` +- Create: `operator/src/main/java/.../api/WorldIngest.java`, `WorldIngestSpec.java`, `WorldIngestStatus.java` +- Test: `operator/src/test/java/.../api/IngestResourceTest.java` +- Modify: `operator/src/test/java/.../CrdGenerationTest.java` + +**Interfaces — bindend, drei Folgeaufgaben bauen darauf:** + +```java +// WorldSourceSpec — alle Gruppen im Feld initialisiert, wie in Phase 2a +String type; // "s3" | "pterodactyl" | "upload" | "push" +S3Source s3 = new S3Source(); // String endpoint; String bucket; String prefix; + // Ref credentialsSecretRef +Pterodactyl pterodactyl = new Pterodactyl(); // String panelUrl; String serverId; + // Ref credentialsSecretRef; String select = "latest" +String poll; // Cron-Ausdruck, nur für Pull-Typen; null = nur manuell +List worlds = new ArrayList<>(); // String name; String layout = "auto" +Retention retention = new Retention(); // int keepVersions = 5 + +// WorldSourceStatus +String lastSeenVersion; +BundleRef latestBundle = new BundleRef(); // String path; String version; List dimensions +String lastPollTime; +List conditions = new ArrayList<>(); + +// WorldIngestSpec +Ref sourceRef = new Ref(); +String sourceVersion; +String worldName; + +// WorldIngestStatus +String phase; // Pending|Extracting|Transforming|Loading|Succeeded|Failed +Progress progress = new Progress(); // double percent; long bytesDone; long bytesTotal +BundleRef bundle = new BundleRef(); +String jobName; String startTime; String completionTime; +List conditions = new ArrayList<>(); +``` + +- [ ] **Step 1: Katalog und Modul anlegen** + +`settings.gradle.kts`: `include(..., "ingest")` und einen S3-Client ergänzen. Wähle bewusst: Das Projekt nutzt bereits `mc` im Runner-Image, aber ein Java-Job braucht eine Bibliothek. Nimm den AWS-SDK-v2-S3-Client (`software.amazon.awssdk:s3`) oder MinIOs Java-Client — recherchiere die aktuelle Version real gegen Maven Central und dokumentiere die Wahl im Report. + +- [ ] **Step 2: Den fehlschlagenden Test schreiben** + +`IngestResourceTest.java` nach dem Muster von `ApusResourceTest` aus Phase 2a: + +```java + @Test + void bothResourcesAreNamespaced() { + assertTrue(Namespaced.class.isAssignableFrom(WorldSource.class)); + assertTrue(Namespaced.class.isAssignableFrom(WorldIngest.class)); + } + + @Test + void specGroupsAreInitialisedSoReconcilersNeverSeeNull() { + WorldSource source = new WorldSource(); + assertNotNull(source.getSpec().getS3()); + assertNotNull(source.getSpec().getPterodactyl()); + assertNotNull(source.getSpec().getWorlds()); + assertNotNull(source.getSpec().getRetention()); + assertNotNull(source.getStatus().getLatestBundle()); + + WorldIngest ingest = new WorldIngest(); + assertNotNull(ingest.getSpec().getSourceRef()); + assertNotNull(ingest.getStatus().getProgress()); + assertNotNull(ingest.getStatus().getBundle()); + } + + @Test + void retentionDefaultsToFiveVersions() { + assertEquals(5, new WorldSource().getSpec().getRetention().getKeepVersions()); + } + + @Test + void layoutDefaultsToAutomaticDetection() { + WorldSource.WorldSelector selector = new WorldSource.WorldSelector(); + assertEquals("auto", selector.getLayout()); + } +``` + +- [ ] **Step 3: Test ausführen, Fehlschlag prüfen, Klassen implementieren** + +Beide Ressourcen mit `implements Namespaced`, Annotationen `@Group("bluemap.onelitefeather.net")`, `@Version("v1alpha1")`, `@Kind`, `@Plural` (`worldsources`, `worldingests`), `@ShortNames` (`bmsource`, `bmingest`), und `initSpec()`/`initStatus()` überschrieben — sonst liefert `new WorldSource().getSpec()` `null`, was in Phase 2a bereits einmal drei parallele Aufgaben blockiert hat. + +- [ ] **Step 4: CRD-Zusicherungen erweitern** + +In `CrdGenerationTest` je einen Test, dass `worldsources` und `worldingests` erzeugt werden und **`scope: Namespaced`** tragen. Nutze die vorhandene, gezielt ladende Hilfsmethode. + +Run: `./gradlew :operator:clean :operator:test` +Expected: PASS, `operator/build/crds/` enthält jetzt fünf CRDs. + +- [ ] **Step 5: Commit** + +```bash +./gradlew spotlessApply +git add -A +git commit -m "feat(ingest): add world source and ingest custom resources" +``` + +--- + +### Task 2: Layout-Erkennung *(parallel mit Task 3 und 4)* + +> Eigener Worktree. Ausschließlich `ingest/src/main/java/.../LayoutDetector.java`, `WorldLayout.java` und die zugehörigen Tests. Keine andere Datei, keine Build-Datei. + +**Das ist der inhaltliche Kern des ETL-Layers.** Die Quellen liefern unterschiedliche Verzeichnisstrukturen; BlueMap braucht pro Karte einen definierten Pfad zur richtigen Dimension. + +| Layout | Erkennungsmerkmal | Abbildung | +|---|---|---| +| `vanilla` | `/region`, `/DIM-1/region`, `/DIM1/region` | direkt | +| `bukkit` | `/region`, `_nether/DIM-1/region`, `_the_end/DIM1/region` | Ordner zusammenführen | +| `nested` | genau ein Unterverzeichnis, darin eines der obigen | Ebene überspringen, erneut prüfen | + +**Interfaces:** +```java +public record WorldLayout(String kind, Map dimensions) {} +// kind: "vanilla" | "bukkit"; dimensions: "overworld"/"the_nether"/"the_end" → Pfad zum region-Verzeichnis + +public final class LayoutDetector { + /** @throws LayoutDetectionException wenn kein bekanntes Layout erkennbar ist */ + public static WorldLayout detect(Path root, String worldName, String forcedLayout); +} +``` + +- [ ] **Step 1: Die fehlschlagenden Tests schreiben** + +Baue die Verzeichnisstrukturen im Test mit `@TempDir` auf — keine Fixture-Dateien nötig, es geht nur um Struktur. + +Testfälle, jeder mit eigener Begründung im Testnamen: +- Vanilla-Layout mit allen drei Dimensionen wird erkannt und korrekt zugeordnet. +- Vanilla-Layout mit **nur** Overworld wird erkannt (kein Nether, kein End — das ist normal). +- Bukkit-Layout mit `world`, `world_nether`, `world_the_end` wird erkannt und auf dieselben logischen Namen abgebildet. +- Ein zusätzlich verschachteltes Verzeichnis (ZIP-Upload-Fall) wird durchschaut. +- Eine Struktur ohne jedes `region`-Verzeichnis schlägt mit `LayoutDetectionException` fehl, und die Meldung nennt die gefundenen Pfade — Raten ist ausdrücklich unerwünscht. +- `forcedLayout = "bukkit"` auf einer Vanilla-Struktur schlägt fehl, statt still etwas Falsches zu liefern. + +- [ ] **Step 2: Implementieren, Tests grün bekommen, committen** + +--- + +### Task 3: Bundle-Writer und Manifest *(parallel mit Task 2 und 4)* + +> Eigener Worktree. Ausschließlich `BundleManifest.java`, `BundleWriter.java`, `S3Client.java` und Tests. + +**Interfaces:** +```java +public record BundleManifest( + int schemaVersion, String tenant, String worldId, String version, + SourceInfo source, String minecraftVersion, + List dimensions, long sizeBytes, Checksums checksums) { + public record SourceInfo(String type, String ref, String detectedLayout) {} + public record DimensionInfo(String id, String path, List regions, int regionCount) {} + public record Checksums(String algorithm, String manifest) {} + public String toJson(); + public static BundleManifest fromJson(String json); +} + +public final class BundleWriter { + public BundleWriter(S3Client s3, String bucket); + /** Writes the bundle, manifest LAST. @return the bundle path */ + public String write(String tenant, String worldId, String version, + WorldLayoutLike layout, ProgressSink progress); +} +``` + +Damit Task 3 nicht auf Task 2 warten muss, nimmt `BundleWriter` eine schmale Schnittstelle entgegen (`WorldLayoutLike` mit `kind()` und `dimensions()`), die Task 2s Record später erfüllt. Definiere sie in deinem eigenen Paket. + +**Tests, die zählen:** +- Das Manifest wird **zuletzt** geschrieben — prüfe die Reihenfolge der Schreibvorgänge über einen Fake-S3-Client, der sie protokolliert. Das ist der Commit-Punkt und die wichtigste Eigenschaft des Bundles. +- Bricht das Schreiben mittendrin ab, existiert **kein** Manifest, das Bundle gilt also als nicht vorhanden. +- Die Regionsliste im Manifest entspricht den tatsächlich geschriebenen `.mca`-Dateien; Koordinaten werden aus dem Dateinamen `r...mca` gelesen. +- Serialisierung und Deserialisierung des Manifests sind verlustfrei. +- Der Fortschritt wird über `ProgressSink` gemeldet, damit der Job ihn nach außen geben kann. + +--- + +### Task 4: Connector-Schnittstelle und die beiden Pull-Quellen *(parallel mit Task 2 und 3)* + +> Eigener Worktree. Ausschließlich `connector/*` und Tests. + +**Interfaces:** +```java +public interface WorldSourceConnector { + String type(); + /** Pull sources list available versions; push sources return an empty list. */ + List discover(Map config); + /** Fetches the raw world data into workDir. */ + void fetch(Map config, SourceVersion version, Path workDir); +} +public record SourceVersion(String id, String label, Instant createdAt, long sizeBytes) {} +``` + +**`S3SourceConnector`:** listet Objekte unter einem Prefix, erkennt neue Versionen anhand des Objektschlüssels, lädt sie herunter. Entpackt gängige Archive, wenn der Schlüssel darauf endet. + +**`PterodactylConnector`:** fragt die Backup-Liste über die Client-API des Panels ab und lädt das gewählte Backup über eine signierte URL. **Recherchiere die tatsächliche API** (Endpunkte, Authentifizierung, Antwortformat) und dokumentiere sie im Report — erfinde keine Endpunkte. Das Backup ist ein `tar.gz` des gesamten Servers; da gzip nicht seekbar ist, wird der Strom **einmal** durchlaufen und dabei selektiv nur das Welt-Verzeichnis geschrieben. Das gesamte Archiv darf nie auf der Platte landen. + +**Tests:** Der S3-Connector gegen einen MinIO-Testcontainer. Der Pterodactyl-Connector gegen einen lokalen HTTP-Stub, der die Panel-Antworten nachbildet — **keinen** echten Panel-Zugriff und keinen Listener auf `0.0.0.0`. Prüfe insbesondere, dass aus einem tar.gz mit Plugins, Configs und Welten nur die Welt-Pfade extrahiert werden. + +--- + +### Task 5: Ingest-Einstiegspunkt und Container-Image + +Analog zu `runner/` aus Phase 1: `IngestMain` liest seine Konfiguration aus Umgebungsvariablen, wählt den Connector, ruft Extract → Detect → Write auf und meldet Fortschritt. Dazu ein `Dockerfile`. + +**Umgebungsvariablen-Vertrag** (die Schnittstelle, die `IngestJobBuilder` in Task 6 bedient): +`APUS_SOURCE_TYPE`, `APUS_WORLD_NAME`, `APUS_LAYOUT` (Default `auto`), `APUS_BUNDLE_BUCKET`, `APUS_BUNDLE_TENANT`, `APUS_BUNDLE_WORLD_ID`, `APUS_BUNDLE_VERSION`, `APUS_S3_ENDPOINT`, `APUS_S3_ACCESS_KEY`, `APUS_S3_SECRET_KEY`, plus die quellenspezifischen (`APUS_SOURCE_S3_*`, `APUS_PTERODACTYL_*`). + +Wie beim Runner: Fehlt eine Pflichtvariable, Abbruch mit klarer Meldung und Exit-Code ungleich null, **bevor** irgendetwas heruntergeladen wird. Nicht-root, `exec` für den Hauptprozess. + +--- + +### Task 6: Reconciler für Quellen und Ingests + +`WorldSourceReconciler`: wertet `poll` aus, vergleicht mit `status.lastSeenVersion`, legt bei Neuem einen `WorldIngest` an. `WorldIngestReconciler`: erzeugt den Job über `IngestJobBuilder`, führt Fortschritt und Ergebnis im Status, setzt bei Erfolg `WorldSource.status.latestBundle`. + +**Bindend:** Eigentümerprüfung wie in Phase 2a. Kein zweiter Ingest für dieselbe Quelle, solange einer läuft — dieselbe optimistische Sperre wie beim Render, dort über `WorldSourceStatus`. Retention: ältere Bundles löschen, aber **nie** eines, das ein `BlueMapRender` noch referenziert. + +--- + +### Task 7: Integrationstest + +Ende-zu-Ende gegen MinIO: eine Welt in Bukkit-Layout als Quelle ablegen, Ingest laufen lassen, prüfen dass ein Bundle mit korrektem Manifest, logisch benannten Dimensionen und vollständiger Regionsliste entsteht. Nutze die vorhandene Fixture `testdata/mini-world`. Eigene `integrationTest`-Task, nicht Teil von `build` — wie in `runner` und `operator`. + +Abschließend: ein Render gegen das erzeugte Bundle starten und belegen, dass der Vertrag zwischen Ingest und Render trägt. + +--- + +## Abschluss Phase 2b + +Danach führt der Weg von einer konfigurierten Quelle bis zur gerenderten Karte ohne Handgriff: `WorldSource` anlegen, Ingest läuft zeitgesteuert, Bundle entsteht, Render startet. + +**Nicht Teil von 2b:** Die Push-Quellen (`upload`, `push`) und das Paper-Plugin — sie folgen in Phase 6. `WorldSource.spec.type` kennt sie bereits, die Connectoren fehlen noch. diff --git a/ingest/Dockerfile b/ingest/Dockerfile new file mode 100644 index 0000000..21257e3 --- /dev/null +++ b/ingest/Dockerfile @@ -0,0 +1,29 @@ +# syntax=docker/dockerfile:1 + +FROM eclipse-temurin:25-jre-jammy + +# Non-root: the ingest job only ever writes below /work (its fetched source data lands in +# /work/source; nothing else on the filesystem is touched). Same convention as runner/Dockerfile. +RUN useradd --uid 10001 --create-home --home-dir /home/apus apus \ + && mkdir -p /work/source \ + && chown -R apus:apus /work + +COPY --chown=apus:apus ingest/entrypoint.sh /opt/apus/entrypoint.sh +# Built by: ./gradlew :ingest:shadowJar +COPY --chown=apus:apus ingest/build/libs/apus-ingest.jar /opt/apus/ingest.jar + +RUN chmod +x /opt/apus/entrypoint.sh + +USER apus +WORKDIR /work + +# Only the genuinely optional variables get a default here. Every required variable +# (APUS_SOURCE_TYPE, APUS_WORLD_NAME, APUS_SOURCE_VERSION, APUS_BUNDLE_*, APUS_S3_ENDPOINT/ +# ACCESS_KEY/SECRET_KEY, and the source-specific ones) intentionally has none, so a Job that +# forgets one fails fast with IngestConfig's error message instead of silently doing the wrong +# thing. See ingest/README.md for the full contract. +ENV APUS_LAYOUT=auto \ + APUS_S3_REGION=us-east-1 \ + APUS_PROGRESS_INTERVAL_SECONDS=10 + +ENTRYPOINT ["/opt/apus/entrypoint.sh"] diff --git a/ingest/README.md b/ingest/README.md new file mode 100644 index 0000000..8cd822e --- /dev/null +++ b/ingest/README.md @@ -0,0 +1,157 @@ +# Apus Ingest Image + +Pulls raw Minecraft world data from a configured source (S3 bucket or a Pterodactyl panel +backup), detects its on-disk layout, and writes it to S3 as a versioned, self-describing world +bundle that the `runner` render container can consume without knowing where the data came from. + +`IngestMain` orchestrates the flow: read and validate configuration from environment variables +(failing before anything is downloaded if one is missing) → fetch via the connector matching +`APUS_SOURCE_TYPE` → detect the layout → write the bundle, manifest last. + +## Build + +```bash +./gradlew :ingest:shadowJar +docker build -f ingest/Dockerfile -t apus/ingest:dev . +``` + +The build context is the repository root, matching `runner/Dockerfile`'s convention -- the image +needs the shadow jar Gradle builds under `ingest/build/libs/`. + +## Run + +```bash +docker run --rm \ + -e APUS_SOURCE_TYPE=s3 \ + -e APUS_WORLD_NAME=world \ + -e APUS_SOURCE_VERSION=2026-08-01T00-00-00Z.zip \ + -e APUS_BUNDLE_BUCKET=bundles \ + -e APUS_BUNDLE_TENANT=acme \ + -e APUS_BUNDLE_SOURCE_NAME=survival-source \ + -e APUS_BUNDLE_WORLD_ID=survival \ + -e APUS_BUNDLE_VERSION=v1 \ + -e APUS_S3_ENDPOINT=http://minio:9000 \ + -e APUS_S3_ACCESS_KEY=... \ + -e APUS_S3_SECRET_KEY=... \ + -e APUS_SOURCE_S3_BUCKET=backups \ + -e APUS_SOURCE_S3_PREFIX=survival/ \ + -e APUS_SOURCE_S3_ACCESS_KEY=... \ + -e APUS_SOURCE_S3_SECRET_KEY=... \ + apus/ingest:dev +``` + +### Environment variables + +The full contract this image accepts, and the interface `IngestJobBuilder` (phase 2b, task 6) +builds Kubernetes Jobs against -- the ingest equivalent of `runner/README.md`'s table. + +| Variable | Required | Default | Meaning | +|---|---|---|---| +| `APUS_SOURCE_TYPE` | yes | — | `s3` or `pterodactyl`. `upload`/`push` are recognised by `WorldSource.spec.type` but have no connector yet (phase 6) -- an unsupported value fails fast rather than being guessed at | +| `APUS_WORLD_NAME` | yes | — | The world's folder name at the source, e.g. `world` | +| `APUS_LAYOUT` | no | `auto` | `auto`, `vanilla`, or `bukkit`. `auto` lets `LayoutDetector` decide; any other value forces that layout and fails detection rather than falling back if the fetched data doesn't actually match it | +| `APUS_SOURCE_VERSION` | yes | — | The exact source version id to fetch, as previously resolved by `WorldSourceReconciler`'s `discover()` poll (task 6) and recorded on the owning `WorldIngest.spec.sourceVersion`. This job never calls `discover()` itself -- see "Design notes" below | +| `APUS_BUNDLE_BUCKET` | yes | — | Destination bucket for the bundle | +| `APUS_BUNDLE_TENANT` | yes | — | Tenant id, becomes the first path segment of the bundle | +| `APUS_BUNDLE_SOURCE_NAME` | yes | — | The owning `WorldSource`'s name, becomes the second path segment. Required so two different sources ingesting a world with the same id (e.g. the vanilla default `world`) never collide on the same bundle path -- see `BundlePath` | +| `APUS_BUNDLE_WORLD_ID` | yes | — | World id, becomes the third path segment | +| `APUS_BUNDLE_VERSION` | yes | — | This bundle version's identifier, becomes the fourth path segment | +| `APUS_S3_ENDPOINT` | yes | — | Bundle destination S3-compatible endpoint, e.g. `http://minio:9000` | +| `APUS_S3_ACCESS_KEY` | yes | — | Bundle destination access key | +| `APUS_S3_SECRET_KEY` | yes | — | Bundle destination secret key | +| `APUS_S3_REGION` | no | `us-east-1` | Bundle destination region | +| `APUS_MC_VERSION` | no | — | Minecraft version recorded as `manifest.minecraftVersion`. Not part of the original task-5 contract -- added because nothing else can supply this value reliably; see "Design notes" | +| `APUS_PROGRESS_INTERVAL_SECONDS` | no | `10` | Minimum seconds between progress lines on stdout; the final update always prints regardless | +| `APUS_MAX_ARCHIVE_TOTAL_BYTES` | no | `5368709120` (5 GiB) | Upper bound on total bytes extracted from one source archive; extraction aborts once exceeded. The work directory has no mounted volume, so this bounds how much of the node's own disk an archive (hostile or just unexpectedly large) can consume -- see `Archives` | +| `APUS_MAX_ARCHIVE_ENTRIES` | no | `200000` | Upper bound on the number of entries (files + directories) extracted from one source archive; extraction aborts once exceeded | +| `APUS_SOURCE_S3_BUCKET` | yes, if `APUS_SOURCE_TYPE=s3` | — | Source bucket | +| `APUS_SOURCE_S3_ENDPOINT` | no | AWS default | Source S3-compatible endpoint | +| `APUS_SOURCE_S3_PREFIX` | no | `""` | Prefix under which each object is one fetchable version | +| `APUS_SOURCE_S3_ACCESS_KEY` | no | credential chain | Source access key; if unset, falls back to the AWS SDK default credentials chain | +| `APUS_SOURCE_S3_SECRET_KEY` | no | credential chain | Source secret key | +| `APUS_SOURCE_S3_REGION` | no | `us-east-1` | Source region | +| `APUS_PTERODACTYL_PANEL_URL` | yes, if `APUS_SOURCE_TYPE=pterodactyl` | — | Panel base URL, e.g. `https://panel.example.com` | +| `APUS_PTERODACTYL_SERVER_ID` | yes, if pterodactyl | — | Server identifier (short id) | +| `APUS_PTERODACTYL_API_KEY` | yes, if pterodactyl | — | Client API key (`ptlc_...`) | +| `APUS_PTERODACTYL_WORLD_PATHS` | yes, if pterodactyl | — | Comma-separated top-level archive paths that make up the world, e.g. `world,world_nether,world_the_end` | + +Missing a required variable (including the source-specific ones for the chosen +`APUS_SOURCE_TYPE`) aborts with a clear `[apus-ingest] ERROR: is required but was not set.` +message on stderr and a non-zero exit, **before** any connector is touched -- see +`IngestConfig.fromEnv` and `IngestMainTest`. + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | Bundle written successfully | +| `1` | Configuration error: a required variable is missing/blank, or `APUS_SOURCE_TYPE` names an unimplemented source. Nothing was fetched. | +| `2` | Layout detection failed -- no known world layout (vanilla/bukkit) could be recognised in the fetched data. The error message names the paths that were actually found. | +| `3` | Any other failure while fetching the source or writing the bundle (network error, S3 error, ...) | + +## Design notes + +**Progress reporting: stdout lines, no HTTP server.** Unlike `runner`, which stays alive for +minutes serving `/progress` to an operator that polls it during a long render, the ingest job is +short-lived and its Kubernetes `Job`/`Pod` status already gives an external reconciler +(`WorldIngestReconciler`, task 6) the coarse state it needs -- `Active`/`Succeeded`/`Failed` plus +timestamps, with no extra moving part in the container. Building an HTTP server here would add a +listening port, a shutdown-ordering concern, and a second thing that can fail, for a job that +typically finishes before a poll loop would notice it existed. Instead, `IngestMain` prints a +`phase=` line at each stage transition +and `ThrottledProgressSink` prints a `progress: NN.N% (done/total bytes)` line at most once per +`APUS_PROGRESS_INTERVAL_SECONDS` (plus unconditionally on the final update) while the bundle is +being written -- exactly the periodic-line-plus-end-state shape the phase 2b plan asks for. If a +future reconciler wants finer-grained percentage rather than just phase, it already has one: read +the pod's logs and parse this same line format, the same relationship `TelemetryContractTest` +documents between `runner`'s log-tail route and its `/progress` endpoint, just without needing an +HTTP round trip at all here. + +**Minecraft version: environment variable, not parsed from `level.dat`.** The Minecraft version a +world was generated/played under lives in `level.dat`'s NBT `Data.Version.Name` (or, on very old +worlds, isn't present at all and must be inferred from `Data.version`, an integer data-version +with its own separate mapping table). Reading it reliably means either adding an NBT-parsing +dependency (none of the ones already in this project's catalog expose it publicly at the ingest +module's layer -- `bluemap-core` has one internally, but `ingest` deliberately does not depend on +BlueMap) or hand-rolling a gzip+NBT reader for a single, easily-gotten-wrong field, for every +supported Minecraft version's `level.dat` shape. Given that `WorldSource`/`WorldIngest` are +already tenant-authored custom resources, this was decided against in favour of a user-supplied +field: `WorldSource.spec.worlds[].minecraftVersion`, which `IngestJobBuilder` reads for the +matching world selector and passes straight through as `APUS_MC_VERSION` -- the tenant already +knows which version they run, and a wrong guess parsed out of `level.dat` would silently mislabel +a manifest forever instead. `APUS_MC_VERSION` itself stays optional at this image's own contract +level (nothing here requires the operator to have set it), and if unset, +`manifest.minecraftVersion` stays `null`, matching `BundleManifest`'s existing "or `null` if not +known at bundle time" contract rather than inventing a new failure mode. If a future task adds +real NBT parsing, it becomes an *additional* fallback ahead of the environment variable, not a +replacement for it -- `level.dat` is genuinely absent for connectors that only fetch specific +region files, so the variable stays useful either way. + +**Source version, not `discover()`, inside the job.** `WorldSourceConnector.discover()` is a +polling operation -- it belongs to `WorldSourceReconciler` (task 6), which resolves "is there a +new version" on a schedule and records the chosen id on `WorldIngest.spec.sourceVersion`. The job +itself only ever calls `fetch()` for the one version it was told to fetch (`APUS_SOURCE_VERSION`); +it never lists what's available. This keeps a single ingest run deterministic and keeps the +"what's new" decision in exactly one place. + +## Integration tests + +`S3SourceConnectorTest` (`ingest/src/test/java/net/onelitefeather/apus/ingest/connector/`) starts +a real MinIO container via Testcontainers and therefore needs Docker. Like `runner` and +`operator` do for their own container-based tests, it is **not** part of `./gradlew build` or +`check` -- it is excluded from the default `test` task and runs only via the explicit task below: + +```bash +./gradlew :ingest:integrationTest +``` + +Every other test in this module (`IngestConfigTest`, `IngestMainTest`, `ThrottledProgressSinkTest`, +`BundleManifestTest`, `BundleWriterTest`, `LayoutDetectorTest`, `S3ClientTest`, +`PterodactylConnectorTest`, `ArchivesTest`, `TarStreamReaderTest`) runs Docker-free as part of the +routine `./gradlew :ingest:test`. + +A full source-to-bundle-to-render end-to-end test (ingest a Bukkit-layout world fixture against +real MinIO, check the resulting manifest, then start a real render against the produced bundle +with the `runner` image) lives in `runner`'s `:runner:integrationTest` +(`IngestRenderContractTest`), not here -- proving the contract between this module's output and +`runner`'s input needs both modules in the same test. diff --git a/ingest/build.gradle.kts b/ingest/build.gradle.kts new file mode 100644 index 0000000..c508d19 --- /dev/null +++ b/ingest/build.gradle.kts @@ -0,0 +1,65 @@ +import java.time.Duration + +plugins { + application + alias(libs.plugins.shadow) +} + +dependencies { + // AWS SDK v2 S3 client -- see settings.gradle.kts for why this over the MinIO Java client. + implementation(platform(libs.aws.sdk.bom)) + implementation(libs.aws.sdk.s3) + + // Jackson -- see settings.gradle.kts for why. Backs BundleManifest (de)serialisation and + // Pterodactyl API response parsing; both used to be hand-rolled JSON codecs. + implementation(platform(libs.jackson.bom)) + implementation(libs.jackson.databind) + + testImplementation(platform(libs.junit.bom)) + testImplementation(libs.junit.jupiter) + testRuntimeOnly(libs.junit.platform.launcher) + + // Real MinIO via Testcontainers for S3SourceConnectorTest -- see that test's Javadoc. + testImplementation(platform(libs.testcontainers.bom)) + testImplementation(libs.testcontainers.junit) + testImplementation(libs.testcontainers.minio) +} + +application { + mainClass.set("net.onelitefeather.apus.ingest.IngestMain") +} + +tasks { + shadowJar { + archiveClassifier.set("") + archiveBaseName.set("apus-ingest") + // Fixed name instead of the default "apus-ingest-.jar": ingest/Dockerfile + // COPYs this file by name (no glob) -- see telemetry-addon/build.gradle.kts for the + // same rationale applied to the render container's addon jar. + archiveFileName.set("apus-ingest.jar") + } + build { + dependsOn(shadowJar) + } +} + +// S3SourceConnectorTest starts a real MinIO container via Testcontainers and therefore needs +// Docker. Exactly like runner/build.gradle.kts and operator/build.gradle.kts do for their own +// container-based tests, that must not run as part of the routine `./gradlew build`/`check` -- +// it would make every build slow and fail outright on a machine without Docker. Excluded from +// the default `test` task and exposed only via the explicit `integrationTest` task below. See +// ingest/README.md for how to run it. +tasks.test { + exclude("**/S3SourceConnectorTest.class") +} + +val integrationTest by tasks.registering(Test::class) { + group = "verification" + description = "Runs S3SourceConnectorTest against a real MinIO container via Testcontainers. " + + "Requires Docker. Not part of build/check." + testClassesDirs = sourceSets.test.get().output.classesDirs + classpath = sourceSets.test.get().runtimeClasspath + include("**/S3SourceConnectorTest.class") + timeout.set(Duration.ofMinutes(5)) + outputs.upToDateWhen { false } +} diff --git a/ingest/entrypoint.sh b/ingest/entrypoint.sh new file mode 100644 index 0000000..d44f605 --- /dev/null +++ b/ingest/entrypoint.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Required configuration is validated by IngestMain itself, before it does any work (see +# IngestConfig.fromEnv) -- this entrypoint stays a thin, replaceable launch line, the same +# separation of concerns runner/entrypoint.sh uses between shell-level checks and the actual +# render process. +# +# No secret ever appears here: JAVA_OPTS carries only JVM tuning flags, and every credential +# IngestMain needs is read straight out of its own environment in-process, never passed as a +# command-line argument to this or any other process (unlike `mc alias set` in +# runner/bin/bundle-sync.sh, which is why that script writes a config file instead). +echo "[apus-ingest] starting" +# shellcheck disable=SC2086 +exec java ${JAVA_OPTS:-} -jar /opt/apus/ingest.jar diff --git a/ingest/src/main/java/net/onelitefeather/apus/ingest/BundleManifest.java b/ingest/src/main/java/net/onelitefeather/apus/ingest/BundleManifest.java new file mode 100644 index 0000000..ffdaac0 --- /dev/null +++ b/ingest/src/main/java/net/onelitefeather/apus/ingest/BundleManifest.java @@ -0,0 +1,125 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.List; + +/** + * The commit point of one version of a world bundle in S3: a versioned, self-describing + * description of an ingested world. + * + *

A bundle is considered to exist only once its manifest object has been written -- see + * {@link BundleWriter}, which writes it strictly after every region file it describes. A reader + * that finds a manifest can therefore trust that every region file it lists is already present; + * the absence of a manifest means the bundle version does not exist, regardless of what else may + * have been left behind by an interrupted write. + * + * @param schemaVersion the manifest schema version, bumped whenever the JSON shape changes + * @param tenant the owning tenant's identifier + * @param worldId the world's identifier within the tenant + * @param version this bundle version's identifier + * @param source where the bundled world data came from + * @param minecraftVersion the Minecraft version the world was generated/played under, or + * {@code null} if not known at bundle time + * @param dimensions every dimension bundled, and the region files each one contains + * @param sizeBytes the total size, in bytes, of every region file written for this bundle version + * @param checksums a content checksum covering all region file bytes written for this bundle + * version + */ +public record BundleManifest( + int schemaVersion, + String tenant, + String worldId, + String version, + SourceInfo source, + String minecraftVersion, + List dimensions, + long sizeBytes, + Checksums checksums) { + + // Jackson deserialises Java records out of the box (via their canonical constructor and + // record-component names, since jackson-databind 2.12 -- no annotations or extra module + // needed), so the record declarations above double as the JSON schema. Enabling + // FAIL_ON_TRAILING_TOKENS is the one piece of non-default configuration this class relies + // on: without it, `readValue` happily accepts and ignores anything after the first JSON + // value, which would make a manifest that got a stray extra document appended to it decode + // as if nothing were wrong. + private static final ObjectMapper MAPPER = + new ObjectMapper().enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS); + + /** + * Where the bundled world data came from. + * + * @param type the source connector type (e.g. {@code "s3"}, {@code "pterodactyl"}), or + * {@code null} if not known to the writer at bundle time + * @param ref an identifier for the exact source version this bundle was produced from + * @param detectedLayout the world layout kind (e.g. {@code "vanilla"} or {@code "bukkit"}) + * that was detected for this world + */ + public record SourceInfo(String type, String ref, String detectedLayout) {} + + /** + * One dimension (overworld, the_nether, the_end, ...) inside the bundle, and the region + * files it contains. + * + * @param id the logical dimension name (e.g. {@code "overworld"}) + * @param path the bundle-relative path this dimension's region files were written under + * @param regions the {@code [x, z]} region coordinates present, read from each region file's + * {@code r...mca} name + * @param regionCount {@code regions.size()}, kept alongside the list so consumers doing a + * quick count/progress check don't need to materialise it + */ + public record DimensionInfo(String id, String path, List regions, int regionCount) {} + + /** + * A content checksum for the bundle. + * + * @param algorithm the digest algorithm used (e.g. {@code "SHA-256"}) + * @param manifest the hex-encoded digest covering all region file bytes written for this + * bundle version + */ + public record Checksums(String algorithm, String manifest) {} + + /** Serialises this manifest to JSON. */ + public String toJson() { + try { + return MAPPER.writeValueAsString(this); + } catch (JsonProcessingException e) { + // A record made up entirely of the types declared above (primitives, strings, + // nested records, List) cannot fail to serialise; this only exists because + // the checked exception has to go somewhere. + throw new IllegalStateException("failed to serialise BundleManifest to JSON", e); + } + } + + /** + * Parses a manifest previously produced by {@link #toJson()}. + * + * @throws IllegalArgumentException if {@code json} is not a valid manifest document + */ + public static BundleManifest fromJson(String json) { + try { + return MAPPER.readValue(json, BundleManifest.class); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException("invalid BundleManifest JSON: " + e.getOriginalMessage(), e); + } + } +} diff --git a/ingest/src/main/java/net/onelitefeather/apus/ingest/BundlePath.java b/ingest/src/main/java/net/onelitefeather/apus/ingest/BundlePath.java new file mode 100644 index 0000000..7804d06 --- /dev/null +++ b/ingest/src/main/java/net/onelitefeather/apus/ingest/BundlePath.java @@ -0,0 +1,59 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest; + +/** + * The single place a bundle's location within the destination bucket is computed from its + * identifying pieces -- {@code tenant}, the {@link net.onelitefeather.apus.ingest.BundleWriter + * source} that produced it, its {@code worldId}, and (for a specific version) its {@code + * version}. + * + *

Why {@code sourceName} is part of the path, not just {@code tenant}/{@code worldId}. + * {@code worldId} is the Minecraft world's own directory name (commonly the vanilla default, + * {@code "world"}), not something the operator guarantees unique -- two {@code WorldSource}s in + * the same namespace can both ingest a world literally named {@code world}. Without a + * source-scoped segment, both sources would write into (and enumerate retention over) the exact + * same bucket prefix, and one source's retention pass could delete the other's still-referenced + * bundle. Kubernetes already guarantees resource names are unique within a namespace, so using the + * owning {@code WorldSource}'s name as a path segment is a free, race-free way to give every + * source its own prefix. + * + *

Before this class existed, this exact three-part concatenation was duplicated across {@link + * BundleWriter}, {@code WorldIngestReconciler}, and {@code AwsBundleStore} (two of them in the + * {@code operator} module, which depends on this one) -- with no {@code sourceName} segment at + * all, which is precisely how the two-sources-same-world-name collision above was possible. One + * shared place for the concatenation makes that class of drift structurally impossible: every + * caller that needs a bundle path calls here instead of rebuilding it locally. + */ +public final class BundlePath { + + private BundlePath() {} + + /** + * The prefix every version of one source's one world is written under, ending in {@code "/"} + * so it can be used directly as an S3 {@code ListObjectsV2} prefix/delimiter query. + */ + public static String prefix(String tenant, String sourceName, String worldId) { + return tenant + "/" + sourceName + "/" + worldId + "/"; + } + + /** One specific bundle version's root path within the bucket. */ + public static String of(String tenant, String sourceName, String worldId, String version) { + return prefix(tenant, sourceName, worldId) + version; + } +} diff --git a/ingest/src/main/java/net/onelitefeather/apus/ingest/BundleWriter.java b/ingest/src/main/java/net/onelitefeather/apus/ingest/BundleWriter.java new file mode 100644 index 0000000..eb9aa4d --- /dev/null +++ b/ingest/src/main/java/net/onelitefeather/apus/ingest/BundleWriter.java @@ -0,0 +1,322 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Writes one version of a world bundle to S3. + * + *

Every region file of every dimension is uploaded first; {@link BundleManifest} is uploaded + * last, and only once every region file has succeeded. S3 offers no cross-object transaction, so + * this write order is what makes the manifest the bundle's commit point: a reader that finds the + * manifest can trust every region file it references already exists, and a write that fails + * partway through never leaves a manifest behind for a bundle version that isn't actually + * complete -- there is nothing to roll back, because the one object that matters was never + * written. + */ +public final class BundleWriter { + + private static final Pattern REGION_FILE_NAME = Pattern.compile("r\\.(-?\\d+)\\.(-?\\d+)\\.mca"); + private static final int SCHEMA_VERSION = 1; + private static final String DIGEST_ALGORITHM = "SHA-256"; + + private final S3Client s3; + private final String bucket; + + public BundleWriter(S3Client s3, String bucket) { + this.s3 = s3; + this.bucket = bucket; + } + + /** + * A narrow view of a detected world layout: just enough for {@link BundleWriter} to write + * one, without depending on the layout-detection module directly. The record the layout + * detector produces is expected to satisfy this interface. + */ + public interface WorldLayoutLike { + /** The detected layout kind, e.g. {@code "vanilla"} or {@code "bukkit"}. */ + String kind(); + + /** Logical dimension name (e.g. {@code "overworld"}) to its region directory. */ + Map dimensions(); + } + + /** Receives progress updates while a bundle is being written. */ + public interface ProgressSink { + /** + * Called after each region file finishes uploading. + * + * @param bytesDone bytes uploaded so far + * @param bytesTotal total bytes to upload, known upfront from the region files on disk + */ + void update(long bytesDone, long bytesTotal); + } + + /** Sibling directories of a dimension's region directory that are part of the bundle when present. */ + private static final String ENTITIES_DIR = "entities"; + + private static final String POI_DIR = "poi"; + private static final String LEVEL_DAT = "level.dat"; + + /** The dimension whose region directory's parent holds the world's {@code level.dat}. */ + private static final String OVERWORLD_DIMENSION = "overworld"; + + /** + * Writes every region file of {@code layout}'s dimensions (plus, where present, each + * dimension's {@code entities}/{@code poi} data and the world's {@code level.dat}), then the + * manifest describing them as the bundle's last object. + * + *

{@code sourceType} and {@code minecraftVersion} are opaque to this class -- neither the + * layout detector nor the bundle writer has any way to know where the data came from or which + * Minecraft version produced it. Only the orchestrator ({@code IngestMain}) knows both, so it + * passes them straight through into {@link BundleManifest#source()} and {@link + * BundleManifest#minecraftVersion()}. + * + *

{@code level.dat} is read from the overworld dimension's region directory's parent (the + * world root every detected layout resolves the overworld's {@code region/} directory + * under -- see {@code LayoutDetector}), since that copy is the one describing the world as a + * whole; the nether/end folders a Bukkit-style split layout produces each carry their own + * {@code level.dat} too, but those are per-dimension placeholders, not the world's actual + * level data. + * + * @param sourceName the {@code WorldSource} this bundle was produced from, by name -- scopes + * the bundle path so two sources whose worlds happen to share a {@code worldId} (e.g. the + * Minecraft default {@code "world"}) never collide on the same prefix; see {@link + * BundlePath} + * @param sourceType the source connector type (e.g. {@code "s3"}, {@code "pterodactyl"}), or + * {@code null} if not known to the caller + * @param sourceRef an identifier for the exact source version this bundle was produced from + * (e.g. a Pterodactyl backup UUID or an S3 object key) -- distinct from {@code version}, + * which is the *bundle's own* version identifier, not where it came from + * @param minecraftVersion the Minecraft version the world was generated/played under, or + * {@code null} if not known to the caller + * @return the bundle's root path within the bucket, {@code ///} + */ + public String write( + String tenant, + String sourceName, + String worldId, + String version, + String sourceType, + String sourceRef, + String minecraftVersion, + WorldLayoutLike layout, + ProgressSink progress) { + String bundlePath = BundlePath.of(tenant, sourceName, worldId, version); + + Map> filesByDimension = new LinkedHashMap<>(); + Map> entityFilesByDimension = new LinkedHashMap<>(); + Map> poiFilesByDimension = new LinkedHashMap<>(); + long totalBytes = 0; + for (Map.Entry entry : layout.dimensions().entrySet()) { + Path regionDir = entry.getValue(); + List files = listRegionFiles(regionDir); + filesByDimension.put(entry.getKey(), files); + for (RegionFile file : files) { + totalBytes += file.sizeBytes(); + } + + List entityFiles = listRegionFilesIfPresent(regionDir.resolveSibling(ENTITIES_DIR)); + entityFilesByDimension.put(entry.getKey(), entityFiles); + for (RegionFile file : entityFiles) { + totalBytes += file.sizeBytes(); + } + + List poiFiles = listRegionFilesIfPresent(regionDir.resolveSibling(POI_DIR)); + poiFilesByDimension.put(entry.getKey(), poiFiles); + for (RegionFile file : poiFiles) { + totalBytes += file.sizeBytes(); + } + } + + Path levelDat = levelDatPath(layout); + long levelDatSize = levelDat != null && Files.isRegularFile(levelDat) ? sizeOrZero(levelDat) : 0; + totalBytes += levelDatSize; + + MessageDigest digest = newDigest(); + List dimensionInfos = new ArrayList<>(); + long bytesDone = 0; + long sizeBytes = 0; + + for (Map.Entry> entry : filesByDimension.entrySet()) { + String dimensionId = entry.getKey(); + String dimensionPath = bundlePath + "/dimensions/" + dimensionId; + List regions = new ArrayList<>(); + for (RegionFile file : entry.getValue()) { + byte[] content = readFully(file.path()); + digest.update(content); + s3.putObject(bucket, dimensionPath + "/region/" + file.path().getFileName(), content); + regions.add(new int[] {file.x(), file.z()}); + sizeBytes += content.length; + bytesDone += content.length; + if (progress != null) { + progress.update(bytesDone, totalBytes); + } + } + dimensionInfos.add( + new BundleManifest.DimensionInfo(dimensionId, dimensionPath, regions, regions.size())); + + bytesDone = writeSidecarFiles( + dimensionPath + "/" + ENTITIES_DIR, + entityFilesByDimension.get(dimensionId), + digest, + progress, + bytesDone, + totalBytes); + bytesDone = writeSidecarFiles( + dimensionPath + "/" + POI_DIR, + poiFilesByDimension.get(dimensionId), + digest, + progress, + bytesDone, + totalBytes); + } + + if (levelDat != null && levelDatSize > 0) { + byte[] content = readFully(levelDat); + digest.update(content); + s3.putObject(bucket, bundlePath + "/" + LEVEL_DAT, content); + sizeBytes += content.length; + bytesDone += content.length; + if (progress != null) { + progress.update(bytesDone, totalBytes); + } + } + + BundleManifest manifest = new BundleManifest( + SCHEMA_VERSION, + tenant, + worldId, + version, + new BundleManifest.SourceInfo(sourceType, sourceRef, layout.kind()), + minecraftVersion, + dimensionInfos, + sizeBytes, + new BundleManifest.Checksums(DIGEST_ALGORITHM, toHex(digest.digest()))); + + // The manifest is the commit point: written last, and only after every region file + // above succeeded. If any putObject or file read above threw, execution never reaches + // this line, and no manifest exists for this bundle version. + s3.putObject(bucket, bundlePath + "/manifest.json", manifest.toJson().getBytes(StandardCharsets.UTF_8)); + + return bundlePath; + } + + /** {@code null} if the layout has no overworld dimension (should not happen for a detected layout). */ + private static Path levelDatPath(WorldLayoutLike layout) { + Path overworldRegion = layout.dimensions().get(OVERWORLD_DIMENSION); + return overworldRegion == null ? null : overworldRegion.resolveSibling(LEVEL_DAT); + } + + private static long sizeOrZero(Path path) { + try { + return Files.size(path); + } catch (IOException e) { + throw new UncheckedIOException("Failed to size " + path, e); + } + } + + /** Uploads {@code files} (entities or poi region files) under {@code targetPrefix}, updating progress as it goes. */ + private long writeSidecarFiles( + String targetPrefix, + List files, + MessageDigest digest, + ProgressSink progress, + long bytesDone, + long totalBytes) { + if (files == null) { + return bytesDone; + } + long done = bytesDone; + for (RegionFile file : files) { + byte[] content = readFully(file.path()); + digest.update(content); + s3.putObject(bucket, targetPrefix + "/" + file.path().getFileName(), content); + done += content.length; + if (progress != null) { + progress.update(done, totalBytes); + } + } + return done; + } + + private record RegionFile(Path path, int x, int z, long sizeBytes) {} + + /** Same as {@link #listRegionFiles}, but returns an empty list rather than failing when {@code dir} does not exist. */ + private static List listRegionFilesIfPresent(Path dir) { + return Files.isDirectory(dir) ? listRegionFiles(dir) : List.of(); + } + + private static List listRegionFiles(Path regionDir) { + List files = new ArrayList<>(); + try (DirectoryStream stream = Files.newDirectoryStream(regionDir, "*.mca")) { + for (Path candidate : stream) { + Matcher matcher = REGION_FILE_NAME.matcher(candidate.getFileName().toString()); + if (!matcher.matches()) { + continue; + } + int x = Integer.parseInt(matcher.group(1)); + int z = Integer.parseInt(matcher.group(2)); + files.add(new RegionFile(candidate, x, z, Files.size(candidate))); + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to list region files in " + regionDir, e); + } + files.sort(Comparator.comparingInt(RegionFile::x).thenComparingInt(RegionFile::z)); + return files; + } + + private static byte[] readFully(Path path) { + try { + return Files.readAllBytes(path); + } catch (IOException e) { + throw new UncheckedIOException("Failed to read region file " + path, e); + } + } + + private static MessageDigest newDigest() { + try { + return MessageDigest.getInstance(DIGEST_ALGORITHM); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(DIGEST_ALGORITHM + " is not available", e); + } + } + + private static String toHex(byte[] bytes) { + StringBuilder sb = new StringBuilder(bytes.length * 2); + for (byte b : bytes) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } +} diff --git a/ingest/src/main/java/net/onelitefeather/apus/ingest/IngestConfig.java b/ingest/src/main/java/net/onelitefeather/apus/ingest/IngestConfig.java new file mode 100644 index 0000000..6a8b8c1 --- /dev/null +++ b/ingest/src/main/java/net/onelitefeather/apus/ingest/IngestConfig.java @@ -0,0 +1,384 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import net.onelitefeather.apus.ingest.connector.PterodactylConnector; +import net.onelitefeather.apus.ingest.connector.S3SourceConnector; + +/** + * The ingest job's complete configuration, read from environment variables and validated eagerly. + * + *

{@link #fromEnv(Map)} is the single place every required variable is checked. It either + * returns a fully valid configuration or throws {@link ConfigurationException} before any network + * call or filesystem write happens -- the job must never partially start (e.g. begin downloading + * from the source) only to fail later because the destination bucket was never configured. + * + *

This is the environment-variable contract {@code IngestJobBuilder} (Task 6) builds Kubernetes + * Jobs against, the same relationship {@code runner/README.md} documents between the render + * container and its operator-built Job. + */ +public final class IngestConfig { + + // -- General -- + public static final String ENV_SOURCE_TYPE = "APUS_SOURCE_TYPE"; + public static final String ENV_WORLD_NAME = "APUS_WORLD_NAME"; + public static final String ENV_LAYOUT = "APUS_LAYOUT"; + public static final String ENV_SOURCE_VERSION = "APUS_SOURCE_VERSION"; + + // -- Bundle destination -- + public static final String ENV_BUNDLE_BUCKET = "APUS_BUNDLE_BUCKET"; + public static final String ENV_BUNDLE_TENANT = "APUS_BUNDLE_TENANT"; + + /** + * The owning {@code WorldSource}'s name -- mandatory, and distinct from {@link + * #ENV_BUNDLE_WORLD_ID}: {@code worldId} is only the Minecraft world's own directory name + * (commonly the vanilla default {@code "world"}), which two different sources in the same + * namespace can share. Scoping the bundle path by source name as well keeps their bundles + * (and, critically, retention passes over them) from ever colliding -- see {@link + * BundlePath}. + */ + public static final String ENV_BUNDLE_SOURCE_NAME = "APUS_BUNDLE_SOURCE_NAME"; + + public static final String ENV_BUNDLE_WORLD_ID = "APUS_BUNDLE_WORLD_ID"; + public static final String ENV_BUNDLE_VERSION = "APUS_BUNDLE_VERSION"; + public static final String ENV_S3_ENDPOINT = "APUS_S3_ENDPOINT"; + public static final String ENV_S3_ACCESS_KEY = "APUS_S3" + "_ACCESS_KEY"; + public static final String ENV_S3_SECRET_KEY = "APUS_S3" + "_SECRET_KEY"; + public static final String ENV_S3_REGION = "APUS_S3_REGION"; + + // -- Not part of the original brief contract, added because the manifest cannot be complete + // without them; see ingest/README.md and task-5-report.md for why each one exists. -- + public static final String ENV_MC_VERSION = "APUS_MC_VERSION"; + public static final String ENV_PROGRESS_INTERVAL_SECONDS = "APUS_PROGRESS_INTERVAL_SECONDS"; + + // -- Archive extraction limits: a zip/tar.gz from an external source is untrusted input; a + // crafted "archive bomb" (either a huge declared/actual uncompressed size, or an enormous + // entry count producing many tiny files) could otherwise fill the ingest job's writable + // container layer -- no volume is mounted for it -- and starve the node it runs on. Both are + // configurable (rather than hardcoded) so an operator can tune them for genuinely large + // worlds without a code change; see Archives.Limits and IngestJobBuilder's `resources`. -- + public static final String ENV_MAX_ARCHIVE_TOTAL_BYTES = "APUS_MAX_ARCHIVE_TOTAL_BYTES"; + public static final String ENV_MAX_ARCHIVE_ENTRIES = "APUS_MAX_ARCHIVE_ENTRIES"; + + // -- Source-specific: s3 -- + public static final String ENV_SOURCE_S3_ENDPOINT = "APUS_SOURCE_S3_ENDPOINT"; + public static final String ENV_SOURCE_S3_BUCKET = "APUS_SOURCE_S3_BUCKET"; + public static final String ENV_SOURCE_S3_PREFIX = "APUS_SOURCE_S3_PREFIX"; + public static final String ENV_SOURCE_S3_ACCESS_KEY = "APUS_SOURCE_S3_ACCESS_KEY"; + public static final String ENV_SOURCE_S3_SECRET_KEY = "APUS_SOURCE_S3_SECRET_KEY"; + public static final String ENV_SOURCE_S3_REGION = "APUS_SOURCE_S3_REGION"; + + // -- Source-specific: pterodactyl -- + public static final String ENV_PTERODACTYL_PANEL_URL = "APUS_PTERODACTYL_PANEL_URL"; + public static final String ENV_PTERODACTYL_SERVER_ID = "APUS_PTERODACTYL_SERVER_ID"; + public static final String ENV_PTERODACTYL_API_KEY = "APUS_PTERODACTYL_API_KEY"; + public static final String ENV_PTERODACTYL_WORLD_PATHS = "APUS_PTERODACTYL_WORLD_PATHS"; + + private static final String TYPE_S3 = "s3"; + private static final String TYPE_PTERODACTYL = "pterodactyl"; + private static final Set SUPPORTED_SOURCE_TYPES = Set.of(TYPE_S3, TYPE_PTERODACTYL); + + private static final String AUTO_LAYOUT = "auto"; + private static final String DEFAULT_S3_REGION = "us-east-1"; + private static final long DEFAULT_PROGRESS_INTERVAL_SECONDS = 10; + + /** 5 GiB -- generous for a real world, still far short of filling a node's disk. */ + private static final long DEFAULT_MAX_ARCHIVE_TOTAL_BYTES = 5L * 1024 * 1024 * 1024; + + /** Generous for even a large multi-dimension world's region/entities/poi file count. */ + private static final long DEFAULT_MAX_ARCHIVE_ENTRIES = 200_000; + + private final String sourceType; + private final String worldName; + private final String forcedLayout; + private final String sourceVersionId; + private final String bundleBucket; + private final String bundleTenant; + private final String bundleSourceName; + private final String bundleWorldId; + private final String bundleVersion; + private final String s3Endpoint; + private final String s3AccessKey; + private final String s3SecretKey; + private final String s3Region; + private final String minecraftVersion; + private final Duration progressInterval; + private final long maxArchiveTotalBytes; + private final long maxArchiveEntries; + private final Map sourceConfig; + + private IngestConfig( + String sourceType, + String worldName, + String forcedLayout, + String sourceVersionId, + String bundleBucket, + String bundleTenant, + String bundleSourceName, + String bundleWorldId, + String bundleVersion, + String s3Endpoint, + String s3AccessKey, + String s3SecretKey, + String s3Region, + String minecraftVersion, + Duration progressInterval, + long maxArchiveTotalBytes, + long maxArchiveEntries, + Map sourceConfig) { + this.sourceType = sourceType; + this.worldName = worldName; + this.forcedLayout = forcedLayout; + this.sourceVersionId = sourceVersionId; + this.bundleBucket = bundleBucket; + this.bundleTenant = bundleTenant; + this.bundleSourceName = bundleSourceName; + this.bundleWorldId = bundleWorldId; + this.bundleVersion = bundleVersion; + this.s3Endpoint = s3Endpoint; + this.s3AccessKey = s3AccessKey; + this.s3SecretKey = s3SecretKey; + this.s3Region = s3Region; + this.minecraftVersion = minecraftVersion; + this.progressInterval = progressInterval; + this.maxArchiveTotalBytes = maxArchiveTotalBytes; + this.maxArchiveEntries = maxArchiveEntries; + this.sourceConfig = sourceConfig; + } + + /** + * Reads and validates every configuration value the ingest job needs from {@code env}. + * + * @throws ConfigurationException if a required variable is missing/blank, or {@code + * APUS_SOURCE_TYPE} names a source this image does not implement + */ + public static IngestConfig fromEnv(Map env) { + String sourceType = requireNonBlank(env, ENV_SOURCE_TYPE); + if (!SUPPORTED_SOURCE_TYPES.contains(sourceType)) { + throw new ConfigurationException("Unsupported " + ENV_SOURCE_TYPE + " '" + sourceType + + "': this image implements only " + SUPPORTED_SOURCE_TYPES + + ". The push sources ('upload', 'push') have no connector yet -- see the phase 2b plan."); + } + + String worldName = requireNonBlank(env, ENV_WORLD_NAME); + String layout = env.getOrDefault(ENV_LAYOUT, AUTO_LAYOUT); + String forcedLayout = AUTO_LAYOUT.equals(layout) ? null : layout; + String sourceVersionId = requireNonBlank(env, ENV_SOURCE_VERSION); + + String bundleBucket = requireNonBlank(env, ENV_BUNDLE_BUCKET); + String bundleTenant = requireNonBlank(env, ENV_BUNDLE_TENANT); + String bundleSourceName = requireNonBlank(env, ENV_BUNDLE_SOURCE_NAME); + String bundleWorldId = requireNonBlank(env, ENV_BUNDLE_WORLD_ID); + String bundleVersion = requireNonBlank(env, ENV_BUNDLE_VERSION); + + String s3Endpoint = requireNonBlank(env, ENV_S3_ENDPOINT); + String s3AccessKey = requireNonBlank(env, ENV_S3_ACCESS_KEY); + String s3SecretKey = requireNonBlank(env, ENV_S3_SECRET_KEY); + String s3Region = env.getOrDefault(ENV_S3_REGION, DEFAULT_S3_REGION); + + String minecraftVersion = blankToNull(env.get(ENV_MC_VERSION)); + Duration progressInterval = Duration.ofSeconds( + parsePositiveLong(env, ENV_PROGRESS_INTERVAL_SECONDS, DEFAULT_PROGRESS_INTERVAL_SECONDS)); + long maxArchiveTotalBytes = + parsePositiveLong(env, ENV_MAX_ARCHIVE_TOTAL_BYTES, DEFAULT_MAX_ARCHIVE_TOTAL_BYTES); + long maxArchiveEntries = parsePositiveLong(env, ENV_MAX_ARCHIVE_ENTRIES, DEFAULT_MAX_ARCHIVE_ENTRIES); + + Map sourceConfig = + switch (sourceType) { + case TYPE_S3 -> s3SourceConfig(env, maxArchiveTotalBytes, maxArchiveEntries); + case TYPE_PTERODACTYL -> pterodactylSourceConfig(env, maxArchiveTotalBytes, maxArchiveEntries); + default -> throw new IllegalStateException("unreachable: " + sourceType); + }; + + return new IngestConfig( + sourceType, + worldName, + forcedLayout, + sourceVersionId, + bundleBucket, + bundleTenant, + bundleSourceName, + bundleWorldId, + bundleVersion, + s3Endpoint, + s3AccessKey, + s3SecretKey, + s3Region, + minecraftVersion, + progressInterval, + maxArchiveTotalBytes, + maxArchiveEntries, + sourceConfig); + } + + private static Map s3SourceConfig( + Map env, long maxArchiveTotalBytes, long maxArchiveEntries) { + Map config = new LinkedHashMap<>(); + config.put(S3SourceConnector.CONFIG_BUCKET, requireNonBlank(env, ENV_SOURCE_S3_BUCKET)); + putIfPresent(config, S3SourceConnector.CONFIG_ENDPOINT, env.get(ENV_SOURCE_S3_ENDPOINT)); + putIfPresent(config, S3SourceConnector.CONFIG_PREFIX, env.get(ENV_SOURCE_S3_PREFIX)); + putIfPresent(config, S3SourceConnector.CONFIG_ACCESS_KEY_ID, env.get(ENV_SOURCE_S3_ACCESS_KEY)); + putIfPresent(config, S3SourceConnector.CONFIG_SECRET_ACCESS_KEY, env.get(ENV_SOURCE_S3_SECRET_KEY)); + putIfPresent(config, S3SourceConnector.CONFIG_REGION, env.get(ENV_SOURCE_S3_REGION)); + putArchiveLimits(config, maxArchiveTotalBytes, maxArchiveEntries); + return config; + } + + private static Map pterodactylSourceConfig( + Map env, long maxArchiveTotalBytes, long maxArchiveEntries) { + Map config = new LinkedHashMap<>(); + config.put(PterodactylConnector.CONFIG_PANEL_URL, requireNonBlank(env, ENV_PTERODACTYL_PANEL_URL)); + config.put(PterodactylConnector.CONFIG_SERVER_ID, requireNonBlank(env, ENV_PTERODACTYL_SERVER_ID)); + config.put(PterodactylConnector.CONFIG_API_KEY, requireNonBlank(env, ENV_PTERODACTYL_API_KEY)); + config.put(PterodactylConnector.CONFIG_WORLD_PATHS, requireNonBlank(env, ENV_PTERODACTYL_WORLD_PATHS)); + putArchiveLimits(config, maxArchiveTotalBytes, maxArchiveEntries); + return config; + } + + private static void putArchiveLimits(Map config, long maxArchiveTotalBytes, long maxArchiveEntries) { + config.put(net.onelitefeather.apus.ingest.connector.Archives.CONFIG_MAX_TOTAL_BYTES, + Long.toString(maxArchiveTotalBytes)); + config.put( + net.onelitefeather.apus.ingest.connector.Archives.CONFIG_MAX_ENTRIES, Long.toString(maxArchiveEntries)); + } + + private static void putIfPresent(Map config, String key, String value) { + if (value != null && !value.isBlank()) { + config.put(key, value); + } + } + + private static String requireNonBlank(Map env, String name) { + String value = env.get(name); + if (value == null || value.isBlank()) { + throw new ConfigurationException(name + " is required but was not set."); + } + return value; + } + + private static String blankToNull(String value) { + return (value == null || value.isBlank()) ? null : value; + } + + private static long parsePositiveLong(Map env, String name, long defaultValue) { + String raw = env.get(name); + if (raw == null || raw.isBlank()) { + return defaultValue; + } + long value; + try { + value = Long.parseLong(raw.trim()); + } catch (NumberFormatException e) { + throw new ConfigurationException(name + " must be a positive integer, got: '" + raw + "'"); + } + if (value <= 0) { + throw new ConfigurationException(name + " must be a positive integer, got: '" + raw + "'"); + } + return value; + } + + public String sourceType() { + return sourceType; + } + + public String worldName() { + return worldName; + } + + /** The layout kind to force detection to, or {@code null} to auto-detect. */ + public String forcedLayout() { + return forcedLayout; + } + + public String sourceVersionId() { + return sourceVersionId; + } + + public String bundleBucket() { + return bundleBucket; + } + + public String bundleTenant() { + return bundleTenant; + } + + /** The owning {@code WorldSource}'s name -- see {@link #ENV_BUNDLE_SOURCE_NAME}. */ + public String bundleSourceName() { + return bundleSourceName; + } + + public String bundleWorldId() { + return bundleWorldId; + } + + public String bundleVersion() { + return bundleVersion; + } + + public String s3Endpoint() { + return s3Endpoint; + } + + public String s3AccessKey() { + return s3AccessKey; + } + + public String s3SecretKey() { + return s3SecretKey; + } + + public String s3Region() { + return s3Region; + } + + /** The Minecraft version to record in the manifest, or {@code null} if not supplied. */ + public String minecraftVersion() { + return minecraftVersion; + } + + public Duration progressInterval() { + return progressInterval; + } + + /** The configured cap on total bytes extracted from one source archive -- see {@link #ENV_MAX_ARCHIVE_TOTAL_BYTES}. */ + public long maxArchiveTotalBytes() { + return maxArchiveTotalBytes; + } + + /** The configured cap on the number of entries extracted from one source archive -- see {@link #ENV_MAX_ARCHIVE_ENTRIES}. */ + public long maxArchiveEntries() { + return maxArchiveEntries; + } + + /** The source-type-specific connector configuration, keyed by each connector's own config keys. */ + public Map sourceConfig() { + return sourceConfig; + } + + /** Thrown when the environment is missing a required variable or holds an invalid value. */ + public static final class ConfigurationException extends RuntimeException { + + ConfigurationException(String message) { + super(message); + } + } +} diff --git a/ingest/src/main/java/net/onelitefeather/apus/ingest/IngestMain.java b/ingest/src/main/java/net/onelitefeather/apus/ingest/IngestMain.java new file mode 100644 index 0000000..18cbce2 --- /dev/null +++ b/ingest/src/main/java/net/onelitefeather/apus/ingest/IngestMain.java @@ -0,0 +1,164 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest; + +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Map; +import net.onelitefeather.apus.ingest.connector.PterodactylConnector; +import net.onelitefeather.apus.ingest.connector.S3SourceConnector; +import net.onelitefeather.apus.ingest.connector.SourceVersion; +import net.onelitefeather.apus.ingest.connector.WorldSourceConnector; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; + +/** + * Entry point of the ingest job: fetch raw world data from one configured source, detect its + * layout, and write it to S3 as a versioned bundle. + * + *

Analogous to {@code runner}'s {@code entrypoint.sh}/BlueMap CLI pairing, but as a single Java + * process rather than a shell script driving a separate JVM: read and validate every required + * environment variable up front (nothing is downloaded until that succeeds), pick the connector + * matching {@code APUS_SOURCE_TYPE}, fetch into a work directory, detect the layout, then hand + * both off to {@link BundleWriter}. See {@code ingest/README.md} for the full environment + * variable contract and exit code meanings. + */ +public final class IngestMain { + + /** Configuration invalid or a required variable missing; nothing was fetched. */ + static final int EXIT_CONFIGURATION_ERROR = 1; + + /** No known world layout could be recognized in the fetched source data. */ + static final int EXIT_LAYOUT_ERROR = 2; + + /** Any other failure while fetching the source or writing the bundle. */ + static final int EXIT_RUNTIME_ERROR = 3; + + private static final int EXIT_SUCCESS = 0; + + private static final Path DEFAULT_WORK_DIR = Path.of("/work/source"); + + private IngestMain() {} + + public static void main(String[] args) { + System.exit(run(System.getenv(), DEFAULT_WORK_DIR)); + } + + /** + * Runs the full ingest flow and returns the process exit code, without calling {@link + * System#exit}. Public so tests can drive it against an arbitrary environment map and work + * directory instead of the real process environment and {@code /work} -- including, from + * {@code runner}'s {@code :runner:integrationTest} task, the end-to-end proof that a bundle + * this method writes is exactly what the render container expects to read (see + * {@code IngestRenderContractTest}). + */ + public static int run(Map env, Path workDir) { + IngestConfig config; + try { + config = IngestConfig.fromEnv(env); + } catch (IngestConfig.ConfigurationException e) { + // Reached before any connector is touched or any directory is created -- see + // IngestConfig.fromEnv's contract. + System.err.println("[apus-ingest] ERROR: " + e.getMessage()); + return EXIT_CONFIGURATION_ERROR; + } + + log("phase=Pending source=%s world=%s bundle=%s/%s/%s" + .formatted( + config.sourceType(), + config.worldName(), + config.bundleTenant(), + config.bundleWorldId(), + config.bundleVersion())); + + try { + Files.createDirectories(workDir); + + log("phase=Extracting"); + WorldSourceConnector connector = selectConnector(config.sourceType()); + SourceVersion version = + new SourceVersion(config.sourceVersionId(), config.sourceVersionId(), Instant.EPOCH, -1); + connector.fetch(config.sourceConfig(), version, workDir); + + log("phase=Transforming"); + WorldLayout layout = LayoutDetector.detect(workDir, config.worldName(), config.forcedLayout()); + log("detected layout kind=%s dimensions=%s".formatted(layout.kind(), layout.dimensions().keySet())); + + log("phase=Loading"); + String bundlePath = writeBundle(config, layout); + + log("phase=Succeeded bundlePath=" + bundlePath); + return EXIT_SUCCESS; + } catch (LayoutDetector.LayoutDetectionException e) { + // The message already names the paths that were actually found -- see + // LayoutDetector's Javadoc: detection fails loudly rather than guessing. + System.err.println("[apus-ingest] phase=Failed ERROR: " + e.getMessage()); + return EXIT_LAYOUT_ERROR; + } catch (Exception e) { + System.err.println("[apus-ingest] phase=Failed ERROR: " + e.getMessage()); + return EXIT_RUNTIME_ERROR; + } + } + + private static String writeBundle(IngestConfig config, WorldLayout layout) { + try (software.amazon.awssdk.services.s3.S3Client awsClient = buildBundleS3Client(config)) { + S3Client bundleS3 = S3Client.wrapping(awsClient); + BundleWriter writer = new BundleWriter(bundleS3, config.bundleBucket()); + ThrottledProgressSink progress = new ThrottledProgressSink(config.progressInterval()); + return writer.write( + config.bundleTenant(), + config.bundleSourceName(), + config.bundleWorldId(), + config.bundleVersion(), + config.sourceType(), + config.sourceVersionId(), + config.minecraftVersion(), + layout, + progress); + } + } + + private static WorldSourceConnector selectConnector(String sourceType) { + return switch (sourceType) { + case "s3" -> new S3SourceConnector(); + case "pterodactyl" -> new PterodactylConnector(); + // IngestConfig.fromEnv already rejects any other value; reaching this would mean the + // two disagree about which source types are supported. + default -> throw new IllegalStateException("unsupported source type: " + sourceType); + }; + } + + private static software.amazon.awssdk.services.s3.S3Client buildBundleS3Client(IngestConfig config) { + return software.amazon.awssdk.services.s3.S3Client.builder() + .region(Region.of(config.s3Region())) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create(config.s3AccessKey(), config.s3SecretKey()))) + .endpointOverride(URI.create(config.s3Endpoint())) + // Bundle destinations are S3-compatible stores (MinIO, Rook/Ceph, ...), not real + // AWS -- see S3SourceConnector.buildClient for the same reasoning. + .forcePathStyle(true) + .build(); + } + + private static void log(String message) { + System.out.println("[apus-ingest] " + message); + } +} diff --git a/ingest/src/main/java/net/onelitefeather/apus/ingest/LayoutDetector.java b/ingest/src/main/java/net/onelitefeather/apus/ingest/LayoutDetector.java new file mode 100644 index 0000000..1a686ab --- /dev/null +++ b/ingest/src/main/java/net/onelitefeather/apus/ingest/LayoutDetector.java @@ -0,0 +1,274 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Detects how a Minecraft world's files are laid out on disk and resolves each dimension to its + * region directory. + * + *

Different ingest sources place the same world's data differently: a vanilla server keeps + * every dimension nested under one world folder ({@code DIM-1} for the nether, {@code DIM1} for + * the end), while a Bukkit-family server splits the nether and the end into sibling folders + * ({@code _nether}, {@code _the_end}). A ZIP upload may additionally wrap the whole + * thing in one extra top-level directory. This class tells those apart so downstream rendering + * always points at the correct region files instead of guessing -- if no known layout can be + * recognized, detection fails loudly rather than picking a plausible-looking wrong answer. + * + *

This class operates on untrusted directory trees. Its input is not first-party data: + * it is extracted Pterodactyl server backups and user-uploaded ZIP archives, both of which are + * fully attacker-controlled. A crafted archive may name a world {@code ../../etc}, or contain a + * symlink such as {@code world/region -> /etc} that a naive walk would happily report as a valid + * dimension path. Whatever this class returns is later read by the bundle writer and uploaded to + * S3, so every returned path is verified -- via real-path resolution -- to stay inside the given + * root, and every directory is checked with {@link LinkOption#NOFOLLOW_LINKS} so symlinks are + * never silently followed. Do not remove these checks to "simplify" traversal; they are the only + * thing standing between a hostile archive and reading/writing arbitrary files on the host. + */ +public final class LayoutDetector { + + private static final String KIND_VANILLA = "vanilla"; + private static final String KIND_BUKKIT = "bukkit"; + + private static final String DIM_OVERWORLD = "overworld"; + private static final String DIM_THE_NETHER = "the_nether"; + private static final String DIM_THE_END = "the_end"; + + private static final String REGION_DIR = "region"; + private static final String NETHER_SUBDIR = "DIM-1"; + private static final String END_SUBDIR = "DIM1"; + + private LayoutDetector() {} + + /** + * Detects the on-disk layout of a world rooted at {@code root}. + * + *

{@code root} is searched directly first. If neither a vanilla nor a bukkit layout is + * found there and {@code root} contains exactly one subdirectory, that subdirectory is + * searched next -- this sees through the extra top-level folder a ZIP upload commonly adds. + * + * @param root the directory to search, e.g. an extracted backup or bucket prefix + * @param worldName the logical world name to look for, e.g. {@code "world"} + * @param forcedLayout when non-null, only that layout kind ({@code "vanilla"} or + * {@code "bukkit"}) is accepted; any other structure fails detection instead of falling + * back to a different kind + * @return the recognized layout with its dimensions resolved to region directories + * @throws LayoutDetectionException if no known layout can be recognized, if {@code worldName} + * is not a single safe path segment, or if resolving a candidate path would escape + * {@code root} (whether via a {@code ..} segment or via a symlink) + */ + public static WorldLayout detect(Path root, String worldName, String forcedLayout) { + validateWorldName(worldName); + Path realRoot = toRealPathOrFail(root); + return detect(root, root, realRoot, worldName, forcedLayout); + } + + /** + * Rejects world names that are not a single, literal path segment. + * + *

The world name comes from a tenant-supplied custom resource and is used verbatim to + * build filesystem paths. Without this check a name such as {@code ../../etc} would let a + * tenant walk the resolved path straight out of the extracted archive root. + */ + private static void validateWorldName(String worldName) { + if (worldName == null || worldName.isEmpty()) { + throw new LayoutDetectionException("World name must not be null or empty."); + } + if (worldName.contains("/") || worldName.contains("\\")) { + throw new LayoutDetectionException( + "World name '" + worldName + "' must not contain path separators."); + } + if (worldName.equals(".") || worldName.equals("..")) { + throw new LayoutDetectionException( + "World name '" + worldName + "' must not be a relative path segment."); + } + } + + private static Path toRealPathOrFail(Path path) { + try { + return path.toRealPath(); + } catch (IOException e) { + throw new LayoutDetectionException("Could not resolve real path of '" + path + "': " + e.getMessage()); + } + } + + private static WorldLayout detect( + Path searchRoot, Path originalRoot, Path realRoot, String worldName, String forcedLayout) { + Optional vanilla = detectVanilla(realRoot, searchRoot, worldName); + Optional bukkit = detectBukkit(realRoot, searchRoot, worldName); + + Optional match = select(vanilla, bukkit, forcedLayout); + if (match.isPresent()) { + return match.get(); + } + + Optional nestedChild = singleSubdirectory(realRoot, searchRoot); + if (nestedChild.isPresent()) { + return detect(nestedChild.get(), originalRoot, realRoot, worldName, forcedLayout); + } + + throw new LayoutDetectionException(failureMessage(originalRoot, searchRoot, worldName, forcedLayout)); + } + + private static Optional select( + Optional vanilla, Optional bukkit, String forcedLayout) { + if (forcedLayout != null) { + return switch (forcedLayout) { + case KIND_VANILLA -> vanilla; + case KIND_BUKKIT -> bukkit; + default -> throw new LayoutDetectionException("Unknown forced layout '" + forcedLayout + + "', expected '" + KIND_VANILLA + "' or '" + KIND_BUKKIT + "'."); + }; + } + if (vanilla.isPresent() && bukkit.isPresent()) { + // Both structurally match on the shared overworld path; the layout that accounts for + // more dimensions is the one that actually explains the directory tree. + return bukkit.get().dimensions().size() > vanilla.get().dimensions().size() ? bukkit : vanilla; + } + return vanilla.isPresent() ? vanilla : bukkit; + } + + private static Optional detectVanilla(Path realRoot, Path searchRoot, String worldName) { + Path worldDir = searchRoot.resolve(worldName); + Path overworld = worldDir.resolve(REGION_DIR); + if (!isRegionDir(realRoot, overworld)) { + return Optional.empty(); + } + Map dimensions = new LinkedHashMap<>(); + dimensions.put(DIM_OVERWORLD, overworld); + putIfRegionDir(realRoot, dimensions, DIM_THE_NETHER, worldDir.resolve(NETHER_SUBDIR).resolve(REGION_DIR)); + putIfRegionDir(realRoot, dimensions, DIM_THE_END, worldDir.resolve(END_SUBDIR).resolve(REGION_DIR)); + return Optional.of(new WorldLayout(KIND_VANILLA, dimensions)); + } + + private static Optional detectBukkit(Path realRoot, Path searchRoot, String worldName) { + Path overworld = searchRoot.resolve(worldName).resolve(REGION_DIR); + if (!isRegionDir(realRoot, overworld)) { + return Optional.empty(); + } + Path netherRegion = + searchRoot.resolve(worldName + "_nether").resolve(NETHER_SUBDIR).resolve(REGION_DIR); + Path endRegion = + searchRoot.resolve(worldName + "_the_end").resolve(END_SUBDIR).resolve(REGION_DIR); + boolean netherPresent = isRegionDir(realRoot, netherRegion); + boolean endPresent = isRegionDir(realRoot, endRegion); + if (!netherPresent && !endPresent) { + // Nothing here distinguishes this from a vanilla layout; do not claim it as bukkit. + return Optional.empty(); + } + Map dimensions = new LinkedHashMap<>(); + dimensions.put(DIM_OVERWORLD, overworld); + if (netherPresent) { + dimensions.put(DIM_THE_NETHER, netherRegion); + } + if (endPresent) { + dimensions.put(DIM_THE_END, endRegion); + } + return Optional.of(new WorldLayout(KIND_BUKKIT, dimensions)); + } + + private static void putIfRegionDir(Path realRoot, Map dimensions, String key, Path candidate) { + if (isRegionDir(realRoot, candidate)) { + dimensions.put(key, candidate); + } + } + + /** + * Returns whether {@code path} is a real, non-symlink directory that resolves to somewhere + * inside {@code realRoot}. + * + *

Two independent checks guard against a hostile archive: {@link LinkOption#NOFOLLOW_LINKS} + * rejects {@code path} outright if it is itself a symlink (even one that would resolve back + * inside the tree -- convenience does not justify the risk), and the real-path containment + * check catches an escape introduced by a symlink higher up the chain, e.g. a world folder + * itself being a symlink to {@code /etc}. + */ + private static boolean isRegionDir(Path realRoot, Path path) { + if (!Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) { + return false; + } + return isWithinRoot(realRoot, path); + } + + private static boolean isWithinRoot(Path realRoot, Path path) { + try { + return path.toRealPath().startsWith(realRoot); + } catch (IOException e) { + return false; + } + } + + private static Optional singleSubdirectory(Path realRoot, Path dir) { + if (!Files.isDirectory(dir, LinkOption.NOFOLLOW_LINKS)) { + return Optional.empty(); + } + try (Stream entries = Files.list(dir)) { + List subdirectories = entries + .filter(p -> Files.isDirectory(p, LinkOption.NOFOLLOW_LINKS)) + .filter(p -> isWithinRoot(realRoot, p)) + .collect(Collectors.toList()); + return subdirectories.size() == 1 ? Optional.of(subdirectories.get(0)) : Optional.empty(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static String failureMessage( + Path originalRoot, Path searchRoot, String worldName, String forcedLayout) { + StringBuilder message = new StringBuilder(); + message.append("Could not recognize a known world layout for world '") + .append(worldName) + .append("' under ") + .append(originalRoot); + if (forcedLayout != null) { + message.append(" (forced layout '").append(forcedLayout).append("')"); + } + message.append(". Checked ").append(searchRoot).append(", found: ").append(listEntries(searchRoot)); + return message.toString(); + } + + private static String listEntries(Path dir) { + if (!Files.isDirectory(dir)) { + return ""; + } + try (Stream entries = Files.list(dir)) { + return entries.map(Path::toString).sorted().collect(Collectors.joining(", ", "[", "]")); + } catch (IOException e) { + return ""; + } + } + + /** Thrown when no known world layout can be recognized under the given root. */ + public static final class LayoutDetectionException extends RuntimeException { + + LayoutDetectionException(String message) { + super(message); + } + } +} diff --git a/ingest/src/main/java/net/onelitefeather/apus/ingest/S3Client.java b/ingest/src/main/java/net/onelitefeather/apus/ingest/S3Client.java new file mode 100644 index 0000000..5a9bde3 --- /dev/null +++ b/ingest/src/main/java/net/onelitefeather/apus/ingest/S3Client.java @@ -0,0 +1,43 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest; + +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; + +/** + * The one S3 write operation {@link BundleWriter} needs, kept deliberately narrow so tests can + * substitute an in-memory fake instead of talking to real S3-compatible storage. + */ +public interface S3Client { + + /** + * Uploads {@code content} to {@code bucket} under {@code key}, overwriting any object + * already there. + */ + void putObject(String bucket, String key, byte[] content); + + /** + * Wraps a real AWS SDK v2 {@link software.amazon.awssdk.services.s3.S3Client} so it can be + * passed to {@link BundleWriter}. + */ + static S3Client wrapping(software.amazon.awssdk.services.s3.S3Client delegate) { + return (bucket, key, content) -> delegate.putObject( + PutObjectRequest.builder().bucket(bucket).key(key).build(), RequestBody.fromBytes(content)); + } +} diff --git a/ingest/src/main/java/net/onelitefeather/apus/ingest/ThrottledProgressSink.java b/ingest/src/main/java/net/onelitefeather/apus/ingest/ThrottledProgressSink.java new file mode 100644 index 0000000..ef0c4d0 --- /dev/null +++ b/ingest/src/main/java/net/onelitefeather/apus/ingest/ThrottledProgressSink.java @@ -0,0 +1,68 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest; + +import java.io.PrintStream; +import java.time.Duration; +import java.time.Instant; +import java.util.Locale; +import java.util.function.Supplier; + +/** + * A {@link BundleWriter.ProgressSink} that prints a plain-text progress line to a stream, at most + * once per {@code minInterval}, plus unconditionally on the final update. + * + *

The ingest job is short-lived and has no HTTP server (unlike {@code runner}'s telemetry + * addon) -- see {@code ingest/README.md} for why a periodic stdout line was chosen instead. {@link + * BundleWriter} may call {@link #update} once per region file, which for a large world can be + * thousands of times in quick succession; without throttling that would flood the job's log + * output without adding useful information. + */ +final class ThrottledProgressSink implements BundleWriter.ProgressSink { + + private final Duration minInterval; + private final Supplier clock; + private final PrintStream out; + private Instant lastReported; + + ThrottledProgressSink(Duration minInterval) { + this(minInterval, Instant::now, System.out); + } + + /** Visible for tests to inject a fake clock and capture output without a real sleep. */ + ThrottledProgressSink(Duration minInterval, Supplier clock, PrintStream out) { + this.minInterval = minInterval; + this.clock = clock; + this.out = out; + this.lastReported = null; + } + + @Override + public void update(long bytesDone, long bytesTotal) { + Instant now = clock.get(); + boolean isFinal = bytesTotal <= 0 || bytesDone >= bytesTotal; + boolean intervalElapsed = + lastReported == null || Duration.between(lastReported, now).compareTo(minInterval) >= 0; + if (!isFinal && !intervalElapsed) { + return; + } + double percent = bytesTotal > 0 ? (100.0 * bytesDone / bytesTotal) : 100.0; + out.printf(Locale.ROOT, "[apus-ingest] progress: %.1f%% (%d/%d bytes)%n", percent, bytesDone, bytesTotal); + lastReported = now; + } +} diff --git a/ingest/src/main/java/net/onelitefeather/apus/ingest/WorldLayout.java b/ingest/src/main/java/net/onelitefeather/apus/ingest/WorldLayout.java new file mode 100644 index 0000000..4fcfdae --- /dev/null +++ b/ingest/src/main/java/net/onelitefeather/apus/ingest/WorldLayout.java @@ -0,0 +1,42 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest; + +import java.nio.file.Path; +import java.util.Map; + +/** + * The recognized on-disk layout of a Minecraft world, with each present dimension resolved to + * the region directory that holds its chunk data. + * + *

Implements {@link BundleWriter.WorldLayoutLike} -- the narrow view {@link BundleWriter} was + * deliberately written against so it could be built in parallel without depending on this class + * directly -- so a detected layout can be handed straight to {@link BundleWriter#write} without + * an adapter. + * + * @param kind the recognized layout kind, either {@code "vanilla"} or {@code "bukkit"} + * @param dimensions logical dimension name ({@code "overworld"}, {@code "the_nether"}, or + * {@code "the_end"}) mapped to the resolved path of its region directory; dimensions that do + * not exist for this world are simply absent from the map + */ +public record WorldLayout(String kind, Map dimensions) implements BundleWriter.WorldLayoutLike { + + public WorldLayout { + dimensions = Map.copyOf(dimensions); + } +} diff --git a/ingest/src/main/java/net/onelitefeather/apus/ingest/connector/Archives.java b/ingest/src/main/java/net/onelitefeather/apus/ingest/connector/Archives.java new file mode 100644 index 0000000..42a9566 --- /dev/null +++ b/ingest/src/main/java/net/onelitefeather/apus/ingest/connector/Archives.java @@ -0,0 +1,258 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest.connector; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Locale; +import java.util.Map; +import java.util.function.Predicate; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +/** + * Extraction helpers for the archive formats a world source may hand back: ZIP and gzip-compressed + * tar. Every entry path is resolved against the target directory with a zip-slip guard, since + * archive contents are attacker-controllable input coming from an external source (an S3 bucket + * or a Pterodactyl panel backup). + * + *

Extraction is bounded by {@link Limits}. The ingest job mounts no volume for its work + * directory and sets no filesystem-level quota of its own -- extraction lands on the container's + * writable layer, backed by the node's own disk. A crafted "archive bomb" (either a huge total + * uncompressed volume, or an enormous number of tiny entries) would otherwise be able to fill that + * disk and, since it is shared with every other pod scheduled on the node, degrade or evict + * unrelated workloads. Both the total-bytes and entry-count limits are enforced as extraction + * proceeds -- not trusted from a declared size in the archive metadata, which is attacker-supplied + * and (for ZIP in particular) not always even present -- so an oversized archive is aborted with a + * clear {@link IOException} partway through rather than only being caught after already having + * written its way past the limit. + */ +public final class Archives { + + /** Config key: total bytes across every extracted entry before extraction aborts. See {@link Limits}. */ + public static final String CONFIG_MAX_TOTAL_BYTES = "archiveMaxTotalBytes"; + + /** Config key: number of entries (files + directories) written before extraction aborts. See {@link Limits}. */ + public static final String CONFIG_MAX_ENTRIES = "archiveMaxEntries"; + + private static final int COPY_BUFFER_SIZE = 8192; + + private Archives() {} + + /** Upper bounds enforced while extracting one archive -- see the class Javadoc. */ + public record Limits(long maxTotalBytes, long maxEntries) { + + /** No limit at all -- only for callers (tests, internal reuse) that intentionally want unbounded extraction. */ + public static final Limits UNBOUNDED = new Limits(Long.MAX_VALUE, Long.MAX_VALUE); + } + + /** + * Reads {@link #CONFIG_MAX_TOTAL_BYTES}/{@link #CONFIG_MAX_ENTRIES} out of a connector's + * config map, as put there by {@code IngestConfig} -- see that class for where the + * configured (or defaulted) values originate. + */ + public static Limits limitsFrom(Map config) { + return new Limits( + parseOrUnbounded(config.get(CONFIG_MAX_TOTAL_BYTES)), parseOrUnbounded(config.get(CONFIG_MAX_ENTRIES))); + } + + private static long parseOrUnbounded(String value) { + if (value == null || value.isBlank()) { + return Long.MAX_VALUE; + } + return Long.parseLong(value.trim()); + } + + static boolean isZip(String key) { + return endsWithIgnoreCase(key, ".zip"); + } + + static boolean isTarGz(String key) { + return endsWithIgnoreCase(key, ".tar.gz") || endsWithIgnoreCase(key, ".tgz"); + } + + static boolean isTar(String key) { + return endsWithIgnoreCase(key, ".tar"); + } + + static boolean isArchive(String key) { + return isZip(key) || isTarGz(key) || isTar(key); + } + + /** Extracts every entry of {@code key}'s archive format from {@code rawIn} into {@code targetDir}, unbounded. */ + static void extract(String key, InputStream rawIn, Path targetDir) throws IOException { + extract(key, rawIn, targetDir, Limits.UNBOUNDED); + } + + /** Same as {@link #extract(String, InputStream, Path)}, bounded by {@code limits}. */ + public static void extract(String key, InputStream rawIn, Path targetDir, Limits limits) throws IOException { + if (isZip(key)) { + extractZip(rawIn, targetDir, limits); + } else if (isTarGz(key)) { + try (GZIPInputStream gzip = new GZIPInputStream(rawIn)) { + extractTar(gzip, targetDir, entryName -> true, limits); + } + } else if (isTar(key)) { + extractTar(rawIn, targetDir, entryName -> true, limits); + } else { + throw new IllegalArgumentException("not a recognised archive key: " + key); + } + } + + private static void extractZip(InputStream in, Path targetDir, Limits limits) throws IOException { + long[] totalBytes = {0}; + long entries = 0; + try (ZipInputStream zip = new ZipInputStream(in)) { + ZipEntry entry; + while ((entry = zip.getNextEntry()) != null) { + entries++; + checkEntryCount(entries, limits); + Path target = resolveSafely(targetDir, entry.getName()); + if (entry.isDirectory()) { + Files.createDirectories(target); + } else { + Files.createDirectories(target.getParent()); + try (OutputStream out = Files.newOutputStream(target)) { + copyLimited(zip, out, limits, totalBytes); + } + } + zip.closeEntry(); + } + } + } + + /** + * Streams an already-decompressed tar body exactly once, writing only entries for which + * {@code include} returns {@code true}. The archive is never buffered as a whole -- an + * included entry is copied straight from the input stream to its target file, and an + * excluded entry's bytes are read and discarded in bounded chunks, never landing on disk. + */ + static void extractTar(InputStream in, Path targetDir, Predicate include) throws IOException { + extractTar(in, targetDir, include, Limits.UNBOUNDED); + } + + /** Same as {@link #extractTar(InputStream, Path, Predicate)}, bounded by {@code limits}. */ + public static void extractTar(InputStream in, Path targetDir, Predicate include, Limits limits) + throws IOException { + long[] totalBytes = {0}; + long entries = 0; + try (TarStreamReader tar = new TarStreamReader(in)) { + TarStreamReader.Entry entry; + while ((entry = tar.nextEntry()) != null) { + if (!include.test(entry.name())) { + continue; + } + entries++; + checkEntryCount(entries, limits); + Path target = resolveSafely(targetDir, entry.name()); + if (entry.directory()) { + Files.createDirectories(target); + continue; + } + Files.createDirectories(target.getParent()); + try (OutputStream out = Files.newOutputStream(target)) { + tar.transferTo(new LimitCheckingOutputStream(out, limits, totalBytes)); + } + } + } + } + + /** + * Copies {@code in} to {@code out} in bounded chunks, tracking {@code totalBytes} (shared + * across every entry of the archive being extracted) against {@code limits.maxTotalBytes()}. + * Enforced against bytes actually copied, not a declared/expected size -- an entry's + * declared size is attacker-controlled input and, for ZIP in particular, not always even + * present -- so a single oversized entry is caught mid-copy exactly like many small entries + * summing past the limit would be. + */ + private static void copyLimited(InputStream in, OutputStream out, Limits limits, long[] totalBytes) + throws IOException { + byte[] buffer = new byte[COPY_BUFFER_SIZE]; + int read; + while ((read = in.read(buffer)) != -1) { + totalBytes[0] += read; + if (totalBytes[0] > limits.maxTotalBytes()) { + throw new IOException("archive exceeds the configured total size limit of " + limits.maxTotalBytes() + + " bytes; aborting extraction"); + } + out.write(buffer, 0, read); + } + } + + /** + * Wraps a destination {@link OutputStream}, enforcing {@code limits.maxTotalBytes()} against a + * running total shared across every entry -- lets {@link TarStreamReader#transferTo} (which + * only knows how to push bytes to an {@code OutputStream}, not report them back in bounded + * chunks) participate in the same total-volume accounting {@link #copyLimited} applies to ZIP. + */ + private static final class LimitCheckingOutputStream extends OutputStream { + private final OutputStream delegate; + private final Limits limits; + private final long[] totalBytes; + + LimitCheckingOutputStream(OutputStream delegate, Limits limits, long[] totalBytes) { + this.delegate = delegate; + this.limits = limits; + this.totalBytes = totalBytes; + } + + @Override + public void write(int b) throws IOException { + checkAndAccount(1); + delegate.write(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + checkAndAccount(len); + delegate.write(b, off, len); + } + + private void checkAndAccount(int additional) throws IOException { + totalBytes[0] += additional; + if (totalBytes[0] > limits.maxTotalBytes()) { + throw new IOException("archive exceeds the configured total size limit of " + limits.maxTotalBytes() + + " bytes; aborting extraction"); + } + } + } + + private static void checkEntryCount(long entries, Limits limits) throws IOException { + if (entries > limits.maxEntries()) { + throw new IOException( + "archive exceeds the configured entry limit of " + limits.maxEntries() + "; aborting extraction"); + } + } + + private static Path resolveSafely(Path targetDir, String entryName) throws IOException { + Path normalisedTargetDir = targetDir.normalize(); + Path target = normalisedTargetDir.resolve(entryName).normalize(); + if (!target.startsWith(normalisedTargetDir)) { + throw new IOException("archive entry escapes target directory: " + entryName); + } + return target; + } + + private static boolean endsWithIgnoreCase(String value, String suffix) { + return value != null && value.toLowerCase(Locale.ROOT).endsWith(suffix); + } +} diff --git a/ingest/src/main/java/net/onelitefeather/apus/ingest/connector/PterodactylConnector.java b/ingest/src/main/java/net/onelitefeather/apus/ingest/connector/PterodactylConnector.java new file mode 100644 index 0000000..2921979 --- /dev/null +++ b/ingest/src/main/java/net/onelitefeather/apus/ingest/connector/PterodactylConnector.java @@ -0,0 +1,287 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest.connector; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Path; +import java.time.Duration; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.zip.GZIPInputStream; + +/** + * A pull source backed by a Pterodactyl game panel's Client + * API. Backups are listed and downloaded exactly the way the Pterodactyl panel's own web UI does: + * {@code GET /api/client/servers/{server}/backups} for the list, then {@code GET + * /api/client/servers/{server}/backups/{backup}/download} for a short-lived signed URL to the + * actual archive. + * + *

API surface, and where it comes from: Pterodactyl's public docs (mirrored under + * pteroapi.com / mintlify) confirm the endpoint shapes, but to pin down exact field names and + * response bodies this implementation was checked directly against the Pterodactyl panel source + * (github.com/pterodactyl/panel, {@code develop} branch, 2026-08-08): + * + *

    + *
  • {@code routes/api-client.php} -- the {@code /servers/{server}/backups} route group: + * {@code GET /} (list), {@code GET /{backup}} (view), {@code GET /{backup}/download} + * (signed URL), plus {@code POST /}, {@code POST /{backup}/lock}, {@code POST + * /{backup}/restore}, {@code DELETE /{backup}} which this connector does not use. + *
  • {@code app/Http/Controllers/Api/Client/Servers/BackupController.php} -- {@code + * index()} paginates with a {@code per_page} query parameter capped at 50 server-side; + * {@code download()} returns {@code new JsonResponse(['object' => 'signed_url', + * 'attributes' => ['url' => $url]])} verbatim, requires the {@code ACTION_BACKUP_DOWNLOAD} + * permission, and only works for backups on the {@code wings} or {@code s3} storage + * adapter. + *
  • {@code app/Transformers/Api/Client/BackupTransformer.php} -- the exact attribute set + * returned per backup: {@code uuid}, {@code is_successful}, {@code is_locked}, {@code + * name}, {@code ignored_files}, {@code checksum}, {@code bytes}, {@code created_at} + * (ISO-8601), {@code completed_at} (ISO-8601 or {@code null}). + *
  • {@code app/Models/ApiKey.php} + {@code app/Providers/AuthServiceProvider.php} -- client + * API keys are Laravel Sanctum personal access tokens prefixed {@code ptlc_}, sent as + * {@code Authorization: Bearer }. + *
+ * + *

The one piece taken from the community docs mirrors rather than the source directly is the + * exact shape of the list endpoint's outer envelope ({@code {"object":"list","data":[...], + * "meta":{"pagination":{...},"backup_count":N}}}) -- this is produced by Pterodactyl's shared + * Fractal serializer, which was not located in this pass, but the envelope is consistent across + * every list endpoint documented for the panel and matches what the mirrors show. If a real panel + * ever disagrees with this envelope, treat that as the fact and this comment as stale. + */ +public final class PterodactylConnector implements WorldSourceConnector { + + public static final String CONFIG_PANEL_URL = "panelUrl"; + public static final String CONFIG_SERVER_ID = "serverId"; + public static final String CONFIG_API_KEY = "apiKey"; + + /** Comma-separated top-level archive paths that make up "the world directory". */ + public static final String CONFIG_WORLD_PATHS = "worldPaths"; + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + /** + * Applied both as the connect timeout (via {@link HttpClient.Builder#connectTimeout}) and as + * the per-request timeout (via {@link HttpRequest.Builder#timeout}) on every call this + * connector makes. Without either, a panel that accepts a TCP connection and then simply + * never responds -- or never finishes sending a large backup body -- would hang the calling + * thread indefinitely. {@code discover()} runs directly inside {@code WorldSourceReconciler}, + * whose JOSDK worker pool is shared across all five reconcilers (see the class Javadoc), so an + * unresponsive panel would starve unrelated render/ingest reconciliation too, not just this + * source's own polling. + */ + private static final Duration REQUEST_TIMEOUT = Duration.ofSeconds(30); + + private final HttpClient httpClient; + + public PterodactylConnector() { + this(HttpClient.newBuilder().connectTimeout(REQUEST_TIMEOUT).build()); + } + + /** Visible for tests to inject a client with tighter timeouts against a local HTTP stub. */ + PterodactylConnector(HttpClient httpClient) { + this.httpClient = httpClient; + } + + @Override + public String type() { + return "pterodactyl"; + } + + @Override + public List discover(Map config) { + String panelUrl = require(config, CONFIG_PANEL_URL); + String serverId = require(config, CONFIG_SERVER_ID); + String apiKey = require(config, CONFIG_API_KEY); + + URI uri = URI.create(trimTrailingSlash(panelUrl) + "/api/client/servers/" + serverId + "/backups?per_page=50"); + HttpResponse response = sendForString(authorizedRequest(uri, apiKey)); + requireSuccess(response, "list backups"); + + JsonNode root = parseJson(response.body()); + requireListEnvelope(root, response.body()); + + List versions = new ArrayList<>(); + for (JsonNode item : root.path("data")) { + JsonNode attributes = item.path("attributes"); + if (!attributes.path("is_successful").asBoolean(false)) { + // A backup still running or that failed has nothing fetchable yet. + continue; + } + String uuid = attributes.path("uuid").asText(null); + String name = attributes.path("name").asText(null); + String createdAt = attributes.path("created_at").asText(null); + long bytes = attributes.path("bytes").asLong(); + versions.add(new SourceVersion(uuid, name, OffsetDateTime.parse(createdAt).toInstant(), bytes)); + } + return versions; + } + + @Override + public void fetch(Map config, SourceVersion version, Path workDir) { + String panelUrl = require(config, CONFIG_PANEL_URL); + String serverId = require(config, CONFIG_SERVER_ID); + String apiKey = require(config, CONFIG_API_KEY); + Set worldPaths = parseWorldPaths(config); + + URI downloadUri = URI.create(trimTrailingSlash(panelUrl) + "/api/client/servers/" + serverId + "/backups/" + + version.id() + "/download"); + HttpResponse signed = sendForString(authorizedRequest(downloadUri, apiKey)); + requireSuccess(signed, "request signed backup download url"); + + JsonNode urlNode = parseJson(signed.body()).path("attributes").path("url"); + if (!urlNode.isTextual()) { + throw new IllegalStateException("Pterodactyl signed_url response had no attributes.url: " + signed.body()); + } + String signedUrl = urlNode.asText(); + + // The backup is a tar.gz of the entire server -- plugins, configs and worlds mixed + // together, potentially tens of gigabytes. gzip is not seekable, so the stream is walked + // exactly once and only entries under the configured world paths are written; the archive + // as a whole is never buffered in memory or written to disk. + URI signedUri = URI.create(signedUrl); + HttpRequest downloadRequest = + HttpRequest.newBuilder(signedUri).timeout(REQUEST_TIMEOUT).GET().build(); + try { + HttpResponse archive = + httpClient.send(downloadRequest, HttpResponse.BodyHandlers.ofInputStream()); + if (archive.statusCode() / 100 != 2) { + throw new IllegalStateException( + "downloading the backup archive failed with HTTP " + archive.statusCode()); + } + try (InputStream raw = archive.body(); + GZIPInputStream gzip = new GZIPInputStream(raw)) { + Archives.extractTar( + gzip, workDir, entryName -> matchesAnyWorldPath(entryName, worldPaths), Archives.limitsFrom(config)); + } + } catch (IOException e) { + // Never embed the signed URL itself in the message: it is a short-lived but + // fully-privileged credential for downloading the entire backup, and exception + // messages end up on stderr and therefore in log aggregation (see IngestMain). The + // host alone is enough to diagnose a connectivity problem. + throw new UncheckedIOException("failed to stream backup archive from host " + signedUri.getHost(), e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted while downloading backup archive", e); + } + } + + private HttpResponse sendForString(HttpRequest request) { + try { + return httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (IOException e) { + throw new UncheckedIOException("Pterodactyl API request failed: " + request.uri(), e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted during Pterodactyl API request: " + request.uri(), e); + } + } + + private static JsonNode parseJson(String body) { + try { + return MAPPER.readTree(body); + } catch (JsonProcessingException e) { + throw new UncheckedIOException("failed to parse Pterodactyl API response as JSON: " + body, e); + } + } + + private static HttpRequest authorizedRequest(URI uri, String apiKey) { + return HttpRequest.newBuilder(uri) + .header("Authorization", "Bearer " + apiKey) + .header("Accept", "application/json") + .timeout(REQUEST_TIMEOUT) + .GET() + .build(); + } + + private static void requireSuccess(HttpResponse response, String action) { + if (response.statusCode() / 100 != 2) { + throw new IllegalStateException("Pterodactyl API call to " + action + " failed with HTTP " + + response.statusCode() + ": " + response.body()); + } + } + + /** + * Rejects a backups-list response that does not match the documented list envelope ({@code + * {"object":"list","data":[...]}} -- see the class Javadoc's "one piece taken from community + * docs" note) instead of silently falling through. + * + *

Without this check, a panel that answers with an unexpected shape -- a different API + * version, a proxy's error page returned with a 2xx status, a permission response that omits + * {@code data} -- would make {@code root.path("data")} resolve to a Jackson {@code + * MissingNode}, which iterates as empty. That reads as "the source has zero backups", a + * perfectly healthy, reportable state ({@code WorldSourceReconciler}'s {@code UP_TO_DATE} + * condition) -- turning a genuinely unverified assumption about the response shape into a + * silent, permanent misdiagnosis instead of a visible, retried failure. + */ + private static void requireListEnvelope(JsonNode root, String rawBody) { + if (!"list".equals(root.path("object").asText(null)) || !root.path("data").isArray()) { + throw new IllegalStateException( + "Pterodactyl backups response did not match the expected {object:\"list\",data:[...]} envelope: " + + rawBody); + } + } + + private static boolean matchesAnyWorldPath(String entryName, Set worldPaths) { + for (String worldPath : worldPaths) { + if (entryName.equals(worldPath) || entryName.startsWith(worldPath + "/")) { + return true; + } + } + return false; + } + + private static Set parseWorldPaths(Map config) { + String raw = require(config, CONFIG_WORLD_PATHS); + Set paths = new LinkedHashSet<>(); + for (String path : raw.split(",")) { + String trimmed = path.trim(); + if (!trimmed.isEmpty()) { + paths.add(trimmed); + } + } + if (paths.isEmpty()) { + throw new IllegalArgumentException("Pterodactyl source config " + CONFIG_WORLD_PATHS + " must list at least one path"); + } + return paths; + } + + private static String trimTrailingSlash(String url) { + return url.endsWith("/") ? url.substring(0, url.length() - 1) : url; + } + + private static String require(Map config, String key) { + String value = config.get(key); + if (value == null || value.isBlank()) { + throw new IllegalArgumentException("missing required Pterodactyl source config key: " + key); + } + return value; + } +} diff --git a/ingest/src/main/java/net/onelitefeather/apus/ingest/connector/S3SourceConnector.java b/ingest/src/main/java/net/onelitefeather/apus/ingest/connector/S3SourceConnector.java new file mode 100644 index 0000000..3decc08 --- /dev/null +++ b/ingest/src/main/java/net/onelitefeather/apus/ingest/connector/S3SourceConnector.java @@ -0,0 +1,164 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest.connector; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.S3Object; + +/** + * A pull source backed by an S3-compatible bucket. Each object found directly under {@code + * config.get(CONFIG_PREFIX)} (no further path segments) is treated as one fetchable version, + * identified by its key relative to the prefix -- new object, new version. If the key ends with a + * recognised archive extension it is unpacked into the work directory; otherwise the raw object is + * written as a single file. + */ +public final class S3SourceConnector implements WorldSourceConnector { + + public static final String CONFIG_ENDPOINT = "endpoint"; + public static final String CONFIG_BUCKET = "bucket"; + public static final String CONFIG_PREFIX = "prefix"; + public static final String CONFIG_ACCESS_KEY_ID = "accessKeyId"; + public static final String CONFIG_SECRET_ACCESS_KEY = "secretAccessKey"; + public static final String CONFIG_REGION = "region"; + + private static final String DEFAULT_REGION = "us-east-1"; + + @Override + public String type() { + return "s3"; + } + + @Override + public List discover(Map config) { + try (S3Client client = buildClient(config)) { + return discover(client, config); + } + } + + @Override + public void fetch(Map config, SourceVersion version, Path workDir) { + try (S3Client client = buildClient(config)) { + fetch(client, config, version, workDir); + } + } + + /** Same as {@link #discover(Map)} but against an already-built client, for testing. */ + List discover(S3Client client, Map config) { + String bucket = require(config, CONFIG_BUCKET); + String prefix = normalisePrefix(config.get(CONFIG_PREFIX)); + + ListObjectsV2Request request = ListObjectsV2Request.builder() + .bucket(bucket) + .prefix(prefix) + .delimiter("/") + .build(); + + List versions = new ArrayList<>(); + for (S3Object object : client.listObjectsV2Paginator(request).contents()) { + String key = object.key(); + if (key.equals(prefix)) { + continue; // a zero-byte "directory marker" object, not a version + } + String id = key.substring(prefix.length()); + versions.add(new SourceVersion(id, id, object.lastModified(), object.size())); + } + return versions; + } + + /** Same as {@link #fetch(Map, SourceVersion, Path)} but against an already-built client, for testing. */ + void fetch(S3Client client, Map config, SourceVersion version, Path workDir) { + String bucket = require(config, CONFIG_BUCKET); + String prefix = normalisePrefix(config.get(CONFIG_PREFIX)); + String key = prefix + version.id(); + + GetObjectRequest request = + GetObjectRequest.builder().bucket(bucket).key(key).build(); + try (ResponseInputStream object = client.getObject(request)) { + if (Archives.isArchive(key)) { + Archives.extract(key, object, workDir, Archives.limitsFrom(config)); + } else { + Path target = workDir.resolve(fileNameOf(key)); + Files.copy(object, target, StandardCopyOption.REPLACE_EXISTING); + } + } catch (IOException e) { + throw new UncheckedIOException("failed to fetch S3 object " + key, e); + } + } + + private static S3Client buildClient(Map config) { + S3ClientBuilder builder = S3Client.builder() + .region(Region.of(config.getOrDefault(CONFIG_REGION, DEFAULT_REGION))) + .credentialsProvider(credentialsProvider(config)); + String endpoint = config.get(CONFIG_ENDPOINT); + if (endpoint != null && !endpoint.isBlank()) { + // S3-compatible stores (MinIO, Rook/Ceph, R2, ...) are reached through an endpoint + // override and need path-style bucket addressing rather than AWS's virtual-hosted + // style, which only real S3 DNS resolves. + builder = builder.endpointOverride(URI.create(endpoint)).forcePathStyle(true); + } + return builder.build(); + } + + private static AwsCredentialsProvider credentialsProvider(Map config) { + String accessKeyId = config.get(CONFIG_ACCESS_KEY_ID); + String secretAccessKey = config.get(CONFIG_SECRET_ACCESS_KEY); + if (accessKeyId != null && secretAccessKey != null) { + return StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey)); + } + return DefaultCredentialsProvider.builder().build(); + } + + private static String normalisePrefix(String prefix) { + if (prefix == null || prefix.isBlank()) { + return ""; + } + return prefix.endsWith("/") ? prefix : prefix + "/"; + } + + private static String fileNameOf(String key) { + int lastSlash = key.lastIndexOf('/'); + return lastSlash < 0 ? key : key.substring(lastSlash + 1); + } + + private static String require(Map config, String key) { + String value = config.get(key); + if (value == null || value.isBlank()) { + throw new IllegalArgumentException("missing required S3 source config key: " + key); + } + return value; + } +} diff --git a/ingest/src/main/java/net/onelitefeather/apus/ingest/connector/SourceVersion.java b/ingest/src/main/java/net/onelitefeather/apus/ingest/connector/SourceVersion.java new file mode 100644 index 0000000..0135d4b --- /dev/null +++ b/ingest/src/main/java/net/onelitefeather/apus/ingest/connector/SourceVersion.java @@ -0,0 +1,34 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest.connector; + +import java.time.Instant; + +/** + * One version of raw world data that a {@link WorldSourceConnector} can fetch: a backup, an + * object generation, or any other source-specific notion of "a point in time we can pull". + * + * @param id source-specific identifier, opaque to callers; passed back into {@link + * WorldSourceConnector#fetch} to fetch this exact version again + * @param label human-readable label for display; may equal {@code id} if the source has nothing + * nicer to offer + * @param createdAt when this version was produced at the source + * @param sizeBytes total size of the raw payload in bytes, or {@code -1} if the source does not + * report it up front + */ +public record SourceVersion(String id, String label, Instant createdAt, long sizeBytes) {} diff --git a/ingest/src/main/java/net/onelitefeather/apus/ingest/connector/TarStreamReader.java b/ingest/src/main/java/net/onelitefeather/apus/ingest/connector/TarStreamReader.java new file mode 100644 index 0000000..339d5fb --- /dev/null +++ b/ingest/src/main/java/net/onelitefeather/apus/ingest/connector/TarStreamReader.java @@ -0,0 +1,254 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest.connector; + +import java.io.Closeable; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; + +/** + * A minimal, single-pass reader for the tar container format (ustar, with GNU long-name and PAX + * extended-header extensions), reading entries directly off an {@link InputStream} without ever + * buffering the archive as a whole. + * + *

This exists only because the ingest module cannot take on a general-purpose archive library + * dependency from within this task's file scope (build files are out of bounds -- see the task + * report). It implements exactly the subset of the format real-world backups use: regular files, + * directories, ustar name prefixes, GNU long names ({@code typeflag 'L'}), and PAX path overrides + * ({@code typeflag 'x'}). It is not a general-purpose tar library and does not attempt to be one. + */ +final class TarStreamReader implements Closeable { + + private static final int BLOCK_SIZE = 512; + + private final InputStream in; + private long remainingInEntry; + private long entryPadding; + private String pendingLongName; + + TarStreamReader(InputStream in) { + this.in = in; + } + + /** One tar entry's header fields relevant to extraction. */ + record Entry(String name, long size, boolean directory) {} + + /** + * Advances to the next entry, skipping whatever of the previous entry (and its block padding) + * was not consumed via {@link #transferTo}. + * + * @return the next entry, or {@code null} at the end of the archive + */ + Entry nextEntry() throws IOException { + finishCurrentEntry(); + while (true) { + byte[] header = readHeaderBlock(); + if (header == null || isAllZero(header)) { + return null; + } + + char typeflag = (char) (header[156] & 0xFF); + long size = parseSize(header, 124, 12); + + if (typeflag == 'L') { + pendingLongName = readAndUnpad(size); + continue; + } + if (typeflag == 'K') { + skipFully(size + paddingFor(size)); + continue; + } + if (typeflag == 'x' || typeflag == 'g') { + String paxPath = parsePaxPath(readAndUnpad(size)); + if (paxPath != null) { + pendingLongName = paxPath; + } + continue; + } + + String name = readString(header, 0, 100); + String prefix = readString(header, 345, 155); + String fullName = prefix.isEmpty() ? name : prefix + "/" + name; + if (pendingLongName != null) { + fullName = pendingLongName; + pendingLongName = null; + } + + remainingInEntry = size; + entryPadding = paddingFor(size); + boolean directory = typeflag == '5' || fullName.endsWith("/"); + return new Entry(fullName, size, directory); + } + } + + /** Copies the current entry's remaining content to {@code out}. */ + long transferTo(OutputStream out) throws IOException { + long copied = 0; + byte[] buffer = new byte[8192]; + while (remainingInEntry > 0) { + int toRead = (int) Math.min(buffer.length, remainingInEntry); + int read = in.read(buffer, 0, toRead); + if (read < 0) { + throw new EOFException("unexpected end of tar stream while reading entry content"); + } + out.write(buffer, 0, read); + remainingInEntry -= read; + copied += read; + } + return copied; + } + + @Override + public void close() throws IOException { + in.close(); + } + + private void finishCurrentEntry() throws IOException { + skipFully(remainingInEntry + entryPadding); + remainingInEntry = 0; + entryPadding = 0; + } + + private String readAndUnpad(long size) throws IOException { + byte[] data = readExact(toIntSize(size)); + skipFully(paddingFor(size)); + return new String(data, StandardCharsets.UTF_8).replace("\0", ""); + } + + private byte[] readHeaderBlock() throws IOException { + byte[] block = new byte[BLOCK_SIZE]; + int total = 0; + while (total < BLOCK_SIZE) { + int read = in.read(block, total, BLOCK_SIZE - total); + if (read < 0) { + if (total == 0) { + return null; + } + throw new EOFException("truncated tar header block"); + } + total += read; + } + return block; + } + + private byte[] readExact(int n) throws IOException { + byte[] data = new byte[n]; + int total = 0; + while (total < n) { + int read = in.read(data, total, n - total); + if (read < 0) { + throw new EOFException("unexpected end of tar stream"); + } + total += read; + } + return data; + } + + private void skipFully(long n) throws IOException { + long remaining = n; + byte[] buffer = new byte[8192]; + while (remaining > 0) { + int toRead = (int) Math.min(buffer.length, remaining); + int read = in.read(buffer, 0, toRead); + if (read < 0) { + throw new EOFException("unexpected end of tar stream while skipping"); + } + remaining -= read; + } + } + + private static long paddingFor(long size) { + return (BLOCK_SIZE - (int) (size % BLOCK_SIZE)) % BLOCK_SIZE; + } + + private static int toIntSize(long size) throws IOException { + if (size < 0 || size > Integer.MAX_VALUE) { + throw new IOException("unsupported tar entry size: " + size); + } + return (int) size; + } + + private static boolean isAllZero(byte[] block) { + for (byte b : block) { + if (b != 0) { + return false; + } + } + return true; + } + + private static String readString(byte[] header, int offset, int length) { + int end = offset; + int limit = offset + length; + while (end < limit && header[end] != 0) { + end++; + } + return new String(header, offset, end - offset, StandardCharsets.UTF_8); + } + + private static long parseSize(byte[] header, int offset, int length) { + if ((header[offset] & 0x80) != 0) { + // GNU base-256 extension: high bit set marks a big-endian binary size, used for + // files too large to fit the 8GiB ceiling of a 12-digit octal field. + long value = 0; + for (int i = 1; i < length; i++) { + value = (value << 8) | (header[offset + i] & 0xFF); + } + return value; + } + String field = new String(header, offset, length, StandardCharsets.US_ASCII) + .replace("\0", "") + .trim(); + return field.isEmpty() ? 0L : Long.parseLong(field, 8); + } + + /** + * Extracts the {@code path} record from a PAX extended header body (records look like + * {@code " =\n"}, length-prefixed and self-describing). + */ + private static String parsePaxPath(String content) { + int index = 0; + while (index < content.length()) { + int spaceIndex = content.indexOf(' ', index); + if (spaceIndex < 0) { + break; + } + int recordLength; + try { + recordLength = Integer.parseInt(content.substring(index, spaceIndex)); + } catch (NumberFormatException e) { + break; + } + int recordEnd = index + recordLength; + if (recordLength <= 0 || recordEnd > content.length()) { + break; + } + // record body is "key=value\n" (trailing newline dropped) + String record = content.substring(spaceIndex + 1, recordEnd - 1); + int equalsIndex = record.indexOf('='); + if (equalsIndex > 0 && "path".equals(record.substring(0, equalsIndex))) { + return record.substring(equalsIndex + 1); + } + index = recordEnd; + } + return null; + } +} diff --git a/ingest/src/main/java/net/onelitefeather/apus/ingest/connector/WorldSourceConnector.java b/ingest/src/main/java/net/onelitefeather/apus/ingest/connector/WorldSourceConnector.java new file mode 100644 index 0000000..c8d1bde --- /dev/null +++ b/ingest/src/main/java/net/onelitefeather/apus/ingest/connector/WorldSourceConnector.java @@ -0,0 +1,57 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest.connector; + +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +/** + * Fetches raw Minecraft world data from one specific kind of source into a local work directory. + * + *

A connector's responsibility ends at "raw bytes under {@code workDir}". It never interprets + * the resulting directory structure -- that is the layout detector's job -- and never writes a + * bundle -- that is the bundle writer's job. Keeping connectors this narrow means a new source + * type costs exactly these two methods. + */ +public interface WorldSourceConnector { + + /** The {@code WorldSourceSpec.type} value this connector handles, e.g. {@code "s3"}. */ + String type(); + + /** + * Lists versions available at the source. Pull sources (S3, Pterodactyl) report every version + * they can currently see; push sources have nothing to discover on their own and return an + * empty list. + * + * @param config source-specific connection details (endpoint, credentials, ...); the set of + * recognised keys is defined by each implementation + */ + List discover(Map config); + + /** + * Fetches the raw data for one version into {@code workDir}. Only raw bytes are placed on + * disk here -- no layout interpretation happens in a connector. + * + * @param config source-specific connection details (endpoint, credentials, ...); the set of + * recognised keys is defined by each implementation + * @param version the version to fetch, as previously returned by {@link #discover} + * @param workDir an existing, writable directory to fetch into + */ + void fetch(Map config, SourceVersion version, Path workDir); +} diff --git a/ingest/src/test/java/net/onelitefeather/apus/ingest/BundleManifestTest.java b/ingest/src/test/java/net/onelitefeather/apus/ingest/BundleManifestTest.java new file mode 100644 index 0000000..8376e9f --- /dev/null +++ b/ingest/src/test/java/net/onelitefeather/apus/ingest/BundleManifestTest.java @@ -0,0 +1,148 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.junit.jupiter.api.Test; + +class BundleManifestTest { + + private static BundleManifest sampleManifest() { + return new BundleManifest( + 1, + "acme", + "spawn-world", + "2026-08-08T12-00-00Z", + new BundleManifest.SourceInfo("s3", "backups/spawn-2026-08-08.zip", "vanilla"), + "1.21.1", + List.of( + new BundleManifest.DimensionInfo( + "overworld", + "acme/spawn-world/2026-08-08T12-00-00Z/dimensions/overworld", + List.of(new int[] {0, 0}, new int[] {-1, 3}), + 2), + new BundleManifest.DimensionInfo( + "the_nether", + "acme/spawn-world/2026-08-08T12-00-00Z/dimensions/the_nether", + List.of(new int[] {0, 0}), + 1)), + 12_345_678L, + new BundleManifest.Checksums("SHA-256", "deadbeef")); + } + + @Test + void roundTripThroughJsonIsLossless() { + BundleManifest original = sampleManifest(); + + String json = original.toJson(); + BundleManifest decoded = BundleManifest.fromJson(json); + + // Records containing arrays (List) don't get a useful equals() from the compiler + // (int[] compares by reference), so re-serialising the decoded value and comparing the + // JSON text proves the round trip is lossless without relying on that broken equality. + assertEquals(json, decoded.toJson()); + + assertEquals(original.schemaVersion(), decoded.schemaVersion()); + assertEquals(original.tenant(), decoded.tenant()); + assertEquals(original.worldId(), decoded.worldId()); + assertEquals(original.version(), decoded.version()); + assertEquals(original.source(), decoded.source()); + assertEquals(original.minecraftVersion(), decoded.minecraftVersion()); + assertEquals(original.sizeBytes(), decoded.sizeBytes()); + assertEquals(original.checksums(), decoded.checksums()); + + assertEquals(2, decoded.dimensions().size()); + assertEquals("overworld", decoded.dimensions().get(0).id()); + assertEquals( + original.dimensions().get(0).path(), + decoded.dimensions().get(0).path()); + assertEquals(2, decoded.dimensions().get(0).regionCount()); + assertArrayEquals( + new int[] {0, 0}, decoded.dimensions().get(0).regions().get(0)); + assertArrayEquals( + new int[] {-1, 3}, decoded.dimensions().get(0).regions().get(1)); + assertArrayEquals( + new int[] {0, 0}, decoded.dimensions().get(1).regions().get(0)); + } + + @Test + void jsonContainsHumanReadableFieldNames() { + String json = sampleManifest().toJson(); + + assertTrue(json.contains("\"schemaVersion\"")); + assertTrue(json.contains("\"tenant\"")); + assertTrue(json.contains("\"worldId\"")); + assertTrue(json.contains("\"dimensions\"")); + assertTrue(json.contains("\"regionCount\"")); + assertTrue(json.contains("\"checksums\"")); + } + + @Test + void nullFieldsRoundTripAsNull() { + BundleManifest withNulls = new BundleManifest( + 1, + "acme", + "spawn-world", + "v1", + new BundleManifest.SourceInfo(null, "v1", "vanilla"), + null, + List.of(), + 0L, + new BundleManifest.Checksums("SHA-256", "e3b0c4")); + + BundleManifest decoded = BundleManifest.fromJson(withNulls.toJson()); + + assertNull(decoded.minecraftVersion()); + assertNull(decoded.source().type()); + assertTrue(decoded.dimensions().isEmpty()); + } + + @Test + void stringsWithSpecialCharactersSurviveTheRoundTrip() { + BundleManifest manifest = new BundleManifest( + 1, + "acme", + "world \"quoted\" \\ name\nwith\ttab", + "v1", + new BundleManifest.SourceInfo("s3", "ref", "vanilla"), + null, + List.of(), + 0L, + new BundleManifest.Checksums("SHA-256", "e3b0c4")); + + BundleManifest decoded = BundleManifest.fromJson(manifest.toJson()); + + assertEquals(manifest.worldId(), decoded.worldId()); + } + + @Test + void fromJsonRejectsNonObjectRoot() { + assertThrows(IllegalArgumentException.class, () -> BundleManifest.fromJson("[1,2,3]")); + } + + @Test + void fromJsonRejectsTrailingContent() { + assertThrows(IllegalArgumentException.class, () -> BundleManifest.fromJson("{}{}")); + } +} diff --git a/ingest/src/test/java/net/onelitefeather/apus/ingest/BundleWriterTest.java b/ingest/src/test/java/net/onelitefeather/apus/ingest/BundleWriterTest.java new file mode 100644 index 0000000..421f631 --- /dev/null +++ b/ingest/src/test/java/net/onelitefeather/apus/ingest/BundleWriterTest.java @@ -0,0 +1,402 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class BundleWriterTest { + + private static final String BUCKET = "test-bucket"; + private static final String SOURCE_NAME = "survival-source"; + + /** Records every {@code putObject} call, in the exact order they happened. */ + private static final class LoggingFakeS3Client implements S3Client { + private final List keysInOrder = new ArrayList<>(); + private final Map objects = new LinkedHashMap<>(); + private final String failOnKeyContaining; + + LoggingFakeS3Client() { + this(null); + } + + LoggingFakeS3Client(String failOnKeyContaining) { + this.failOnKeyContaining = failOnKeyContaining; + } + + @Override + public void putObject(String bucket, String key, byte[] content) { + if (failOnKeyContaining != null && key.contains(failOnKeyContaining)) { + throw new RuntimeException("simulated failure writing " + key); + } + keysInOrder.add(key); + objects.put(key, content); + } + } + + private static final class RecordingProgressSink implements BundleWriter.ProgressSink { + private final List updates = new ArrayList<>(); + + @Override + public void update(long bytesDone, long bytesTotal) { + updates.add(new long[] {bytesDone, bytesTotal}); + } + } + + private record FakeLayout(String kind, Map dimensions) implements BundleWriter.WorldLayoutLike {} + + private static void writeRegionFile(Path dir, String name, byte[] content) throws IOException { + Files.createDirectories(dir); + Files.write(dir.resolve(name), content); + } + + @Test + void manifestIsWrittenLastAfterEveryRegionFile(@TempDir Path tempDir) throws IOException { + Path overworld = tempDir.resolve("overworld"); + Path nether = tempDir.resolve("nether"); + writeRegionFile(overworld, "r.0.0.mca", "ow-0-0".getBytes(StandardCharsets.UTF_8)); + writeRegionFile(overworld, "r.-1.3.mca", "ow-neg1-3".getBytes(StandardCharsets.UTF_8)); + writeRegionFile(nether, "r.0.0.mca", "nether-0-0".getBytes(StandardCharsets.UTF_8)); + + Map dimensions = new LinkedHashMap<>(); + dimensions.put("overworld", overworld); + dimensions.put("the_nether", nether); + FakeLayout layout = new FakeLayout("vanilla", dimensions); + + LoggingFakeS3Client fake = new LoggingFakeS3Client(); + BundleWriter writer = new BundleWriter(fake, BUCKET); + + String bundlePath = writer.write("acme", SOURCE_NAME, "spawn", "v1", "s3", "src-ref-1", "1.21.10", layout, null); + + assertEquals("acme/" + SOURCE_NAME + "/spawn/v1", bundlePath); + assertEquals(4, fake.keysInOrder.size(), "3 region files + 1 manifest"); + + String manifestKey = bundlePath + "/manifest.json"; + assertEquals(manifestKey, fake.keysInOrder.get(fake.keysInOrder.size() - 1), "manifest must be written last"); + + // Every other key must be a region file key, and none of them may be the manifest. + for (int i = 0; i < fake.keysInOrder.size() - 1; i++) { + String key = fake.keysInOrder.get(i); + assertTrue(key.contains("/region/r."), "expected a region file key, got: " + key); + assertFalse(key.equals(manifestKey)); + } + } + + @Test + void aFailureWritingARegionFileLeavesNoManifestBehind(@TempDir Path tempDir) throws IOException { + Path overworld = tempDir.resolve("overworld"); + writeRegionFile(overworld, "r.0.0.mca", "ow-0-0".getBytes(StandardCharsets.UTF_8)); + writeRegionFile(overworld, "r.0.1.mca", "ow-0-1".getBytes(StandardCharsets.UTF_8)); + + Map dimensions = new LinkedHashMap<>(); + dimensions.put("overworld", overworld); + FakeLayout layout = new FakeLayout("vanilla", dimensions); + + // Fails on the second region file (r.0.1.mca), simulating a mid-write network error. + LoggingFakeS3Client fake = new LoggingFakeS3Client("r.0.1.mca"); + BundleWriter writer = new BundleWriter(fake, BUCKET); + + assertThrows( + RuntimeException.class, + () -> writer.write("acme", SOURCE_NAME, "spawn", "v1", "s3", "src-ref-1", "1.21.10", layout, null)); + + assertTrue(fake.objects.keySet().stream().noneMatch(key -> key.endsWith("manifest.json")), + "no manifest may exist after a failed write: " + fake.objects.keySet()); + // The manifest write is never even attempted; write() must fail before reaching it. + assertEquals(1, fake.keysInOrder.size(), "only the first region file should have been written"); + } + + @Test + void manifestRegionListMatchesTheActualMcaFilesOnDisk(@TempDir Path tempDir) throws IOException { + Path overworld = tempDir.resolve("overworld"); + writeRegionFile(overworld, "r.0.0.mca", "a".getBytes(StandardCharsets.UTF_8)); + writeRegionFile(overworld, "r.2.-5.mca", "bb".getBytes(StandardCharsets.UTF_8)); + writeRegionFile(overworld, "r.-3.-3.mca", "ccc".getBytes(StandardCharsets.UTF_8)); + // A non-region file in the same directory must be ignored. + Files.write(overworld.resolve("session.lock"), "lock".getBytes(StandardCharsets.UTF_8)); + + Map dimensions = new LinkedHashMap<>(); + dimensions.put("overworld", overworld); + FakeLayout layout = new FakeLayout("vanilla", dimensions); + + LoggingFakeS3Client fake = new LoggingFakeS3Client(); + BundleWriter writer = new BundleWriter(fake, BUCKET); + + String bundlePath = writer.write("acme", SOURCE_NAME, "spawn", "v1", "s3", "src-ref-1", "1.21.10", layout, null); + + byte[] manifestBytes = fake.objects.get(bundlePath + "/manifest.json"); + BundleManifest manifest = BundleManifest.fromJson(new String(manifestBytes, StandardCharsets.UTF_8)); + + assertEquals(1, manifest.dimensions().size()); + BundleManifest.DimensionInfo overworldInfo = manifest.dimensions().get(0); + assertEquals("overworld", overworldInfo.id()); + assertEquals(3, overworldInfo.regionCount()); + assertEquals(3, overworldInfo.regions().size()); + + List regions = overworldInfo.regions(); + assertArrayEquals(new int[] {-3, -3}, regions.get(0)); + assertArrayEquals(new int[] {0, 0}, regions.get(1)); + assertArrayEquals(new int[] {2, -5}, regions.get(2)); + + assertEquals(6L, manifest.sizeBytes(), "1 + 2 + 3 bytes across the three region files"); + assertEquals("vanilla", manifest.source().detectedLayout()); + } + + @Test + void progressIsReportedAsRegionFilesAreWritten(@TempDir Path tempDir) throws IOException { + Path overworld = tempDir.resolve("overworld"); + writeRegionFile(overworld, "r.0.0.mca", new byte[10]); + writeRegionFile(overworld, "r.0.1.mca", new byte[20]); + + Map dimensions = new LinkedHashMap<>(); + dimensions.put("overworld", overworld); + FakeLayout layout = new FakeLayout("vanilla", dimensions); + + LoggingFakeS3Client fake = new LoggingFakeS3Client(); + RecordingProgressSink progress = new RecordingProgressSink(); + BundleWriter writer = new BundleWriter(fake, BUCKET); + + writer.write("acme", SOURCE_NAME, "spawn", "v1", "s3", "src-ref-1", "1.21.10", layout, progress); + + assertEquals(2, progress.updates.size()); + long[] first = progress.updates.get(0); + long[] second = progress.updates.get(1); + assertEquals(30L, first[1], "total bytes is known upfront from the files on disk"); + assertEquals(30L, second[1]); + assertEquals(30L, second[0], "final update reports every byte as done"); + assertTrue(first[0] <= second[0]); + } + + @Test + void writeToleratesANullProgressSink(@TempDir Path tempDir) throws IOException { + Path overworld = tempDir.resolve("overworld"); + writeRegionFile(overworld, "r.0.0.mca", "x".getBytes(StandardCharsets.UTF_8)); + + Map dimensions = new LinkedHashMap<>(); + dimensions.put("overworld", overworld); + FakeLayout layout = new FakeLayout("vanilla", dimensions); + + BundleWriter writer = new BundleWriter(new LoggingFakeS3Client(), BUCKET); + + String bundlePath = writer.write("acme", SOURCE_NAME, "spawn", "v1", "s3", "src-ref-1", "1.21.10", layout, null); + + assertEquals("acme/" + SOURCE_NAME + "/spawn/v1", bundlePath); + } + + @Test + void manifestRecordsTheSourceTypeAndMinecraftVersionSuppliedByTheCaller(@TempDir Path tempDir) + throws IOException { + Path overworld = tempDir.resolve("overworld"); + writeRegionFile(overworld, "r.0.0.mca", "a".getBytes(StandardCharsets.UTF_8)); + + Map dimensions = new LinkedHashMap<>(); + dimensions.put("overworld", overworld); + FakeLayout layout = new FakeLayout("bukkit", dimensions); + + LoggingFakeS3Client fake = new LoggingFakeS3Client(); + BundleWriter writer = new BundleWriter(fake, BUCKET); + + String bundlePath = + writer.write("acme", SOURCE_NAME, "spawn", "v1", "pterodactyl", "backup-uuid-1", "1.20.4", layout, null); + + byte[] manifestBytes = fake.objects.get(bundlePath + "/manifest.json"); + BundleManifest manifest = BundleManifest.fromJson(new String(manifestBytes, StandardCharsets.UTF_8)); + + assertEquals("pterodactyl", manifest.source().type()); + assertEquals("bukkit", manifest.source().detectedLayout()); + assertEquals("1.20.4", manifest.minecraftVersion()); + } + + @Test + void aNullSourceTypeAndMinecraftVersionAreToleratedAndRoundTripAsNull(@TempDir Path tempDir) throws IOException { + Path overworld = tempDir.resolve("overworld"); + writeRegionFile(overworld, "r.0.0.mca", "a".getBytes(StandardCharsets.UTF_8)); + + Map dimensions = new LinkedHashMap<>(); + dimensions.put("overworld", overworld); + FakeLayout layout = new FakeLayout("vanilla", dimensions); + + LoggingFakeS3Client fake = new LoggingFakeS3Client(); + BundleWriter writer = new BundleWriter(fake, BUCKET); + + writer.write("acme", SOURCE_NAME, "spawn", "v1", null, null, null, layout, null); + + byte[] manifestBytes = fake.objects.get("acme/" + SOURCE_NAME + "/spawn/v1/manifest.json"); + BundleManifest manifest = BundleManifest.fromJson(new String(manifestBytes, StandardCharsets.UTF_8)); + + assertNull(manifest.source().type()); + assertNull(manifest.minecraftVersion()); + } + + // --- C2: bundle path is scoped by source name ------------------------------------------- + + @Test + void twoSourcesWithTheSameWorldIdNeverShareABundlePath(@TempDir Path tempDir) throws IOException { + Path overworld = tempDir.resolve("overworld"); + writeRegionFile(overworld, "r.0.0.mca", "a".getBytes(StandardCharsets.UTF_8)); + Map dimensions = new LinkedHashMap<>(); + dimensions.put("overworld", overworld); + FakeLayout layout = new FakeLayout("vanilla", dimensions); + + LoggingFakeS3Client fake = new LoggingFakeS3Client(); + BundleWriter writer = new BundleWriter(fake, BUCKET); + + // Same tenant, same worldId ("world", the Minecraft default), same version -- only the + // owning source differs, exactly the collision scenario the fix guards against. + String pathA = writer.write("acme", "source-a", "world", "v1", "s3", "ref", "1.21.10", layout, null); + String pathB = writer.write("acme", "source-b", "world", "v1", "s3", "ref", "1.21.10", layout, null); + + assertFalse(pathA.equals(pathB), "two different sources must never resolve to the same bundle path"); + assertEquals("acme/source-a/world/v1", pathA); + assertEquals("acme/source-b/world/v1", pathB); + } + + // --- D2: manifest.source.ref carries the source's own version identifier ---------------- + + @Test + void manifestSourceRefIsTheSuppliedSourceVersionIdentifierNotTheBundleVersion(@TempDir Path tempDir) + throws IOException { + Path overworld = tempDir.resolve("overworld"); + writeRegionFile(overworld, "r.0.0.mca", "a".getBytes(StandardCharsets.UTF_8)); + Map dimensions = new LinkedHashMap<>(); + dimensions.put("overworld", overworld); + FakeLayout layout = new FakeLayout("vanilla", dimensions); + + LoggingFakeS3Client fake = new LoggingFakeS3Client(); + BundleWriter writer = new BundleWriter(fake, BUCKET); + + String bundlePath = writer.write( + "acme", + SOURCE_NAME, + "spawn", + "ingest-run-7", // the bundle's own version identifier + "pterodactyl", + "11111111-1111-1111-1111-111111111111", // the actual Pterodactyl backup UUID + "1.21.10", + layout, + null); + + byte[] manifestBytes = fake.objects.get(bundlePath + "/manifest.json"); + BundleManifest manifest = BundleManifest.fromJson(new String(manifestBytes, StandardCharsets.UTF_8)); + + assertEquals( + "11111111-1111-1111-1111-111111111111", + manifest.source().ref(), + "source.ref must be the source's own version identifier, not the bundle version"); + assertFalse( + "ingest-run-7".equals(manifest.source().ref()), + "source.ref must not be the bundle version, which describes this bundle, not where it came from"); + } + + // --- D1: level.dat, entities/ and poi/ are included when present ------------------------ + + @Test + void writeIncludesLevelDatFromTheOverworldDirectory(@TempDir Path tempDir) throws IOException { + Path overworld = tempDir.resolve("world/region"); + writeRegionFile(overworld, "r.0.0.mca", "a".getBytes(StandardCharsets.UTF_8)); + Files.writeString(overworld.getParent().resolve("level.dat"), "level-data"); + + Map dimensions = new LinkedHashMap<>(); + dimensions.put("overworld", overworld); + FakeLayout layout = new FakeLayout("vanilla", dimensions); + + LoggingFakeS3Client fake = new LoggingFakeS3Client(); + BundleWriter writer = new BundleWriter(fake, BUCKET); + + String bundlePath = writer.write("acme", SOURCE_NAME, "spawn", "v1", "s3", "ref", "1.21.10", layout, null); + + byte[] levelDat = fake.objects.get(bundlePath + "/level.dat"); + assertEquals("level-data", new String(levelDat, StandardCharsets.UTF_8)); + } + + @Test + void writeToleratesAMissingLevelDat(@TempDir Path tempDir) throws IOException { + Path overworld = tempDir.resolve("world/region"); + writeRegionFile(overworld, "r.0.0.mca", "a".getBytes(StandardCharsets.UTF_8)); + // No level.dat written next to it. + + Map dimensions = new LinkedHashMap<>(); + dimensions.put("overworld", overworld); + FakeLayout layout = new FakeLayout("vanilla", dimensions); + + LoggingFakeS3Client fake = new LoggingFakeS3Client(); + BundleWriter writer = new BundleWriter(fake, BUCKET); + + String bundlePath = writer.write("acme", SOURCE_NAME, "spawn", "v1", "s3", "ref", "1.21.10", layout, null); + + assertFalse(fake.objects.containsKey(bundlePath + "/level.dat")); + } + + @Test + void writeIncludesEntitiesAndPoiFilesWhenPresent(@TempDir Path tempDir) throws IOException { + Path overworld = tempDir.resolve("world/region"); + writeRegionFile(overworld, "r.0.0.mca", "region-data".getBytes(StandardCharsets.UTF_8)); + writeRegionFile(overworld.getParent().resolve("entities"), "r.0.0.mca", "entity-data".getBytes(StandardCharsets.UTF_8)); + writeRegionFile(overworld.getParent().resolve("poi"), "r.0.0.mca", "poi-data".getBytes(StandardCharsets.UTF_8)); + + Map dimensions = new LinkedHashMap<>(); + dimensions.put("overworld", overworld); + FakeLayout layout = new FakeLayout("vanilla", dimensions); + + LoggingFakeS3Client fake = new LoggingFakeS3Client(); + BundleWriter writer = new BundleWriter(fake, BUCKET); + + String bundlePath = writer.write("acme", SOURCE_NAME, "spawn", "v1", "s3", "ref", "1.21.10", layout, null); + + String dimensionPath = bundlePath + "/dimensions/overworld"; + assertEquals( + "entity-data", + new String(fake.objects.get(dimensionPath + "/entities/r.0.0.mca"), StandardCharsets.UTF_8)); + assertEquals( + "poi-data", new String(fake.objects.get(dimensionPath + "/poi/r.0.0.mca"), StandardCharsets.UTF_8)); + } + + @Test + void writeToleratesMissingEntitiesAndPoiDirectories(@TempDir Path tempDir) throws IOException { + Path overworld = tempDir.resolve("world/region"); + writeRegionFile(overworld, "r.0.0.mca", "region-data".getBytes(StandardCharsets.UTF_8)); + // No entities/ or poi/ siblings created -- e.g. a pre-1.17 world. + + Map dimensions = new LinkedHashMap<>(); + dimensions.put("overworld", overworld); + FakeLayout layout = new FakeLayout("vanilla", dimensions); + + LoggingFakeS3Client fake = new LoggingFakeS3Client(); + BundleWriter writer = new BundleWriter(fake, BUCKET); + + String bundlePath = writer.write("acme", SOURCE_NAME, "spawn", "v1", "s3", "ref", "1.21.10", layout, null); + + assertTrue(fake.objects.keySet().stream().noneMatch(key -> key.contains("/entities/"))); + assertTrue(fake.objects.keySet().stream().noneMatch(key -> key.contains("/poi/"))); + } +} diff --git a/ingest/src/test/java/net/onelitefeather/apus/ingest/IngestConfigTest.java b/ingest/src/test/java/net/onelitefeather/apus/ingest/IngestConfigTest.java new file mode 100644 index 0000000..9e80612 --- /dev/null +++ b/ingest/src/test/java/net/onelitefeather/apus/ingest/IngestConfigTest.java @@ -0,0 +1,226 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; +import net.onelitefeather.apus.ingest.connector.PterodactylConnector; +import net.onelitefeather.apus.ingest.connector.S3SourceConnector; +import org.junit.jupiter.api.Test; + +class IngestConfigTest { + + private static Map minimalS3Env() { + Map env = new LinkedHashMap<>(); + env.put(IngestConfig.ENV_SOURCE_TYPE, "s3"); + env.put(IngestConfig.ENV_WORLD_NAME, "world"); + env.put(IngestConfig.ENV_SOURCE_VERSION, "2026-08-01T00-00-00Z.zip"); + env.put(IngestConfig.ENV_BUNDLE_BUCKET, "bundles"); + env.put(IngestConfig.ENV_BUNDLE_TENANT, "acme"); + env.put(IngestConfig.ENV_BUNDLE_SOURCE_NAME, "survival-source"); + env.put(IngestConfig.ENV_BUNDLE_WORLD_ID, "spawn"); + env.put(IngestConfig.ENV_BUNDLE_VERSION, "v1"); + env.put(IngestConfig.ENV_S3_ENDPOINT, "http://minio:9000"); + env.put(IngestConfig.ENV_S3_ACCESS_KEY, "access"); + env.put(IngestConfig.ENV_S3_SECRET_KEY, "secret"); + env.put(IngestConfig.ENV_SOURCE_S3_BUCKET, "worlds"); + return env; + } + + @Test + void everyRequiredFieldPresentProducesAValidConfig() { + IngestConfig config = IngestConfig.fromEnv(minimalS3Env()); + + assertEquals("s3", config.sourceType()); + assertEquals("world", config.worldName()); + assertNull(config.forcedLayout(), "auto is the default and translates to 'let the detector decide'"); + assertEquals("acme", config.bundleTenant()); + assertEquals("spawn", config.bundleWorldId()); + assertEquals("v1", config.bundleVersion()); + assertEquals("us-east-1", config.s3Region(), "unset region falls back to the same default as runner"); + assertNull(config.minecraftVersion(), "not part of the minimal env, must stay null rather than guessed"); + assertEquals(Duration.ofSeconds(10), config.progressInterval()); + } + + @Test + void anExplicitLayoutOtherThanAutoIsPassedThroughAsTheForcedLayout() { + Map env = minimalS3Env(); + env.put(IngestConfig.ENV_LAYOUT, "bukkit"); + + IngestConfig config = IngestConfig.fromEnv(env); + + assertEquals("bukkit", config.forcedLayout()); + } + + @Test + void missingSourceTypeAbortsBeforeAnythingElseIsChecked() { + Map env = minimalS3Env(); + env.remove(IngestConfig.ENV_SOURCE_TYPE); + + IngestConfig.ConfigurationException e = + assertThrows(IngestConfig.ConfigurationException.class, () -> IngestConfig.fromEnv(env)); + assertTrue(e.getMessage().contains(IngestConfig.ENV_SOURCE_TYPE)); + } + + @Test + void blankValuesAreTreatedAsMissing() { + Map env = minimalS3Env(); + env.put(IngestConfig.ENV_WORLD_NAME, " "); + + assertThrows(IngestConfig.ConfigurationException.class, () -> IngestConfig.fromEnv(env)); + } + + @Test + void anUnsupportedSourceTypeIsRejectedRatherThanGuessedAt() { + Map env = minimalS3Env(); + env.put(IngestConfig.ENV_SOURCE_TYPE, "upload"); + + IngestConfig.ConfigurationException e = + assertThrows(IngestConfig.ConfigurationException.class, () -> IngestConfig.fromEnv(env)); + assertTrue(e.getMessage().contains("upload")); + } + + @Test + void missingBundleDestinationFieldsAreDetectedEvenWhenSourceConfigIsComplete() { + Map env = minimalS3Env(); + env.remove(IngestConfig.ENV_BUNDLE_BUCKET); + + IngestConfig.ConfigurationException e = + assertThrows(IngestConfig.ConfigurationException.class, () -> IngestConfig.fromEnv(env)); + assertTrue(e.getMessage().contains(IngestConfig.ENV_BUNDLE_BUCKET)); + } + + @Test + void s3SourceConfigMapsEnvVarsToTheConnectorsOwnConfigKeys() { + Map env = minimalS3Env(); + env.put(IngestConfig.ENV_SOURCE_S3_ENDPOINT, "http://source-minio:9000"); + env.put(IngestConfig.ENV_SOURCE_S3_PREFIX, "backups/"); + env.put(IngestConfig.ENV_SOURCE_S3_ACCESS_KEY, "src-access"); + env.put(IngestConfig.ENV_SOURCE_S3_SECRET_KEY, "src-secret"); + env.put(IngestConfig.ENV_SOURCE_S3_REGION, "eu-central-1"); + + Map sourceConfig = IngestConfig.fromEnv(env).sourceConfig(); + + assertEquals("worlds", sourceConfig.get(S3SourceConnector.CONFIG_BUCKET)); + assertEquals("http://source-minio:9000", sourceConfig.get(S3SourceConnector.CONFIG_ENDPOINT)); + assertEquals("backups/", sourceConfig.get(S3SourceConnector.CONFIG_PREFIX)); + assertEquals("src-access", sourceConfig.get(S3SourceConnector.CONFIG_ACCESS_KEY_ID)); + assertEquals("src-secret", sourceConfig.get(S3SourceConnector.CONFIG_SECRET_ACCESS_KEY)); + assertEquals("eu-central-1", sourceConfig.get(S3SourceConnector.CONFIG_REGION)); + } + + @Test + void missingSourceSpecificFieldIsDetectedForPterodactylToo() { + Map env = minimalS3Env(); + env.put(IngestConfig.ENV_SOURCE_TYPE, "pterodactyl"); + env.put(IngestConfig.ENV_PTERODACTYL_PANEL_URL, "https://panel.example.com"); + env.put(IngestConfig.ENV_PTERODACTYL_SERVER_ID, "abc123"); + env.put(IngestConfig.ENV_PTERODACTYL_WORLD_PATHS, "world,world_nether,world_the_end"); + // APUS_PTERODACTYL_API_KEY intentionally left unset. + + IngestConfig.ConfigurationException e = + assertThrows(IngestConfig.ConfigurationException.class, () -> IngestConfig.fromEnv(env)); + assertTrue(e.getMessage().contains(IngestConfig.ENV_PTERODACTYL_API_KEY)); + } + + @Test + void pterodactylSourceConfigMapsEnvVarsToTheConnectorsOwnConfigKeys() { + Map env = minimalS3Env(); + env.put(IngestConfig.ENV_SOURCE_TYPE, "pterodactyl"); + env.put(IngestConfig.ENV_PTERODACTYL_PANEL_URL, "https://panel.example.com"); + env.put(IngestConfig.ENV_PTERODACTYL_SERVER_ID, "abc123"); + env.put(IngestConfig.ENV_PTERODACTYL_API_KEY, "ptlc_secret"); + env.put(IngestConfig.ENV_PTERODACTYL_WORLD_PATHS, "world,world_nether,world_the_end"); + + Map sourceConfig = IngestConfig.fromEnv(env).sourceConfig(); + + assertEquals("https://panel.example.com", sourceConfig.get(PterodactylConnector.CONFIG_PANEL_URL)); + assertEquals("abc123", sourceConfig.get(PterodactylConnector.CONFIG_SERVER_ID)); + assertEquals("ptlc_secret", sourceConfig.get(PterodactylConnector.CONFIG_API_KEY)); + assertEquals("world,world_nether,world_the_end", sourceConfig.get(PterodactylConnector.CONFIG_WORLD_PATHS)); + } + + @Test + void minecraftVersionIsPassedThroughWhenSupplied() { + Map env = minimalS3Env(); + env.put(IngestConfig.ENV_MC_VERSION, "1.21.10"); + + assertEquals("1.21.10", IngestConfig.fromEnv(env).minecraftVersion()); + } + + @Test + void missingBundleSourceNameIsDetectedEvenWhenEverythingElseIsComplete() { + Map env = minimalS3Env(); + env.remove(IngestConfig.ENV_BUNDLE_SOURCE_NAME); + + IngestConfig.ConfigurationException e = + assertThrows(IngestConfig.ConfigurationException.class, () -> IngestConfig.fromEnv(env)); + assertTrue(e.getMessage().contains(IngestConfig.ENV_BUNDLE_SOURCE_NAME)); + } + + @Test + void archiveLimitsDefaultWhenNotConfigured() { + IngestConfig config = IngestConfig.fromEnv(minimalS3Env()); + + assertTrue(config.maxArchiveTotalBytes() > 0); + assertTrue(config.maxArchiveEntries() > 0); + assertEquals( + Long.toString(config.maxArchiveTotalBytes()), + config.sourceConfig().get(net.onelitefeather.apus.ingest.connector.Archives.CONFIG_MAX_TOTAL_BYTES)); + assertEquals( + Long.toString(config.maxArchiveEntries()), + config.sourceConfig().get(net.onelitefeather.apus.ingest.connector.Archives.CONFIG_MAX_ENTRIES)); + } + + @Test + void archiveLimitsAreConfigurableAndFlowIntoTheSourceConfig() { + Map env = minimalS3Env(); + env.put(IngestConfig.ENV_MAX_ARCHIVE_TOTAL_BYTES, "1024"); + env.put(IngestConfig.ENV_MAX_ARCHIVE_ENTRIES, "7"); + + IngestConfig config = IngestConfig.fromEnv(env); + + assertEquals(1024L, config.maxArchiveTotalBytes()); + assertEquals(7L, config.maxArchiveEntries()); + assertEquals( + "1024", + config.sourceConfig().get(net.onelitefeather.apus.ingest.connector.Archives.CONFIG_MAX_TOTAL_BYTES)); + assertEquals( + "7", config.sourceConfig().get(net.onelitefeather.apus.ingest.connector.Archives.CONFIG_MAX_ENTRIES)); + } + + @Test + void progressIntervalMustBeAPositiveInteger() { + Map env = minimalS3Env(); + env.put(IngestConfig.ENV_PROGRESS_INTERVAL_SECONDS, "not-a-number"); + + assertThrows(IngestConfig.ConfigurationException.class, () -> IngestConfig.fromEnv(env)); + + env.put(IngestConfig.ENV_PROGRESS_INTERVAL_SECONDS, "0"); + assertThrows(IngestConfig.ConfigurationException.class, () -> IngestConfig.fromEnv(env)); + + env.put(IngestConfig.ENV_PROGRESS_INTERVAL_SECONDS, "30"); + assertEquals(Duration.ofSeconds(30), IngestConfig.fromEnv(env).progressInterval()); + } +} diff --git a/ingest/src/test/java/net/onelitefeather/apus/ingest/IngestMainTest.java b/ingest/src/test/java/net/onelitefeather/apus/ingest/IngestMainTest.java new file mode 100644 index 0000000..8cbb990 --- /dev/null +++ b/ingest/src/test/java/net/onelitefeather/apus/ingest/IngestMainTest.java @@ -0,0 +1,73 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Covers the one part of {@link IngestMain#run} that is meaningfully testable without a real + * source and a real S3 endpoint: that a missing or invalid required variable aborts cleanly, + * with a non-zero exit code, before any directory is even created -- the "no half-started job" + * requirement from the phase 2b plan. The full happy path (fetch -> detect -> write), plus the + * proof that the resulting bundle is exactly what the {@code runner} render container expects to + * read, is exercised end to end against MinIO and the real {@code apus/runner} image in {@code + * runner}'s {@code :runner:integrationTest} task ({@code IngestRenderContractTest}) -- not here, + * since proving the ingest/render contract needs both modules together. + */ +class IngestMainTest { + + @Test + void missingRequiredVariableExitsNonZeroAndTouchesNothing(@TempDir Path tempDir) { + Path workDir = tempDir.resolve("source"); + Map env = new LinkedHashMap<>(); // completely empty + + int exitCode = IngestMain.run(env, workDir); + + assertEquals(IngestMain.EXIT_CONFIGURATION_ERROR, exitCode); + assertFalse(Files.exists(workDir), "nothing may be fetched before configuration is fully valid"); + } + + @Test + void unsupportedSourceTypeExitsNonZeroAndTouchesNothing(@TempDir Path tempDir) { + Path workDir = tempDir.resolve("source"); + Map env = new LinkedHashMap<>(); + env.put(IngestConfig.ENV_SOURCE_TYPE, "upload"); + env.put(IngestConfig.ENV_WORLD_NAME, "world"); + env.put(IngestConfig.ENV_SOURCE_VERSION, "v1"); + env.put(IngestConfig.ENV_BUNDLE_BUCKET, "bundles"); + env.put(IngestConfig.ENV_BUNDLE_TENANT, "acme"); + env.put(IngestConfig.ENV_BUNDLE_WORLD_ID, "spawn"); + env.put(IngestConfig.ENV_BUNDLE_VERSION, "v1"); + env.put(IngestConfig.ENV_S3_ENDPOINT, "http://minio:9000"); + env.put(IngestConfig.ENV_S3_ACCESS_KEY, "access"); + env.put(IngestConfig.ENV_S3_SECRET_KEY, "secret"); + + int exitCode = IngestMain.run(env, workDir); + + assertEquals(IngestMain.EXIT_CONFIGURATION_ERROR, exitCode); + assertFalse(Files.exists(workDir)); + } +} diff --git a/ingest/src/test/java/net/onelitefeather/apus/ingest/LayoutDetectorTest.java b/ingest/src/test/java/net/onelitefeather/apus/ingest/LayoutDetectorTest.java new file mode 100644 index 0000000..cdedf9f --- /dev/null +++ b/ingest/src/test/java/net/onelitefeather/apus/ingest/LayoutDetectorTest.java @@ -0,0 +1,174 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import net.onelitefeather.apus.ingest.LayoutDetector.LayoutDetectionException; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class LayoutDetectorTest { + + @Test + void vanillaLayoutWithAllThreeDimensionsIsRecognizedAndMappedCorrectly(@TempDir Path root) throws IOException { + Path world = root.resolve("world"); + createRegionDir(world); + createRegionDir(world.resolve("DIM-1")); + createRegionDir(world.resolve("DIM1")); + + WorldLayout layout = LayoutDetector.detect(root, "world", null); + + assertEquals("vanilla", layout.kind()); + assertEquals(world.resolve("region"), layout.dimensions().get("overworld")); + assertEquals(world.resolve("DIM-1").resolve("region"), layout.dimensions().get("the_nether")); + assertEquals(world.resolve("DIM1").resolve("region"), layout.dimensions().get("the_end")); + } + + @Test + void vanillaLayoutWithOnlyOverworldIsRecognizedSinceMissingNetherAndEndIsNormal(@TempDir Path root) + throws IOException { + Path world = root.resolve("world"); + createRegionDir(world); + + WorldLayout layout = LayoutDetector.detect(root, "world", null); + + assertEquals("vanilla", layout.kind()); + assertEquals(world.resolve("region"), layout.dimensions().get("overworld")); + assertFalse(layout.dimensions().containsKey("the_nether")); + assertFalse(layout.dimensions().containsKey("the_end")); + } + + @Test + void bukkitLayoutWithSiblingWorldFoldersIsRecognizedAndMappedToLogicalDimensionNames(@TempDir Path root) + throws IOException { + createRegionDir(root.resolve("world")); + createRegionDir(root.resolve("world_nether").resolve("DIM-1")); + createRegionDir(root.resolve("world_the_end").resolve("DIM1")); + + WorldLayout layout = LayoutDetector.detect(root, "world", null); + + assertEquals("bukkit", layout.kind()); + assertEquals( + root.resolve("world").resolve("region"), layout.dimensions().get("overworld")); + assertEquals( + root.resolve("world_nether").resolve("DIM-1").resolve("region"), + layout.dimensions().get("the_nether")); + assertEquals( + root.resolve("world_the_end").resolve("DIM1").resolve("region"), + layout.dimensions().get("the_end")); + } + + @Test + void extraWrappingDirectoryFromAZipUploadIsSeenThrough(@TempDir Path root) throws IOException { + Path wrapper = root.resolve("upload-a1b2c3"); + Path world = wrapper.resolve("world"); + createRegionDir(world); + createRegionDir(world.resolve("DIM-1")); + + WorldLayout layout = LayoutDetector.detect(root, "world", null); + + assertEquals("vanilla", layout.kind()); + assertEquals(world.resolve("region"), layout.dimensions().get("overworld")); + assertEquals(world.resolve("DIM-1").resolve("region"), layout.dimensions().get("the_nether")); + } + + @Test + void structureWithoutAnyRegionDirectoryFailsAndMessageNamesTheFoundPaths(@TempDir Path root) throws IOException { + Path world = root.resolve("world"); + Path playerdata = world.resolve("playerdata"); + Path stats = world.resolve("stats"); + Files.createDirectories(playerdata); + Files.createDirectories(stats); + + LayoutDetectionException exception = + assertThrows(LayoutDetectionException.class, () -> LayoutDetector.detect(root, "world", null)); + + assertTrue(exception.getMessage().contains(playerdata.toString()), exception.getMessage()); + assertTrue(exception.getMessage().contains(stats.toString()), exception.getMessage()); + } + + @Test + void forcingBukkitLayoutOnAVanillaStructureFailsInsteadOfSilentlyReturningTheWrongLayout(@TempDir Path root) + throws IOException { + Path world = root.resolve("world"); + createRegionDir(world); + createRegionDir(world.resolve("DIM-1")); + createRegionDir(world.resolve("DIM1")); + + assertThrows( + LayoutDetectionException.class, () -> LayoutDetector.detect(root, "world", "bukkit")); + } + + @Test + void worldNameContainingDotDotSegmentsFailsInsteadOfEscapingRoot(@TempDir Path root) { + LayoutDetectionException exception = assertThrows( + LayoutDetectionException.class, () -> LayoutDetector.detect(root, "../../etc", null)); + + assertTrue(exception.getMessage().contains("path separators"), exception.getMessage()); + } + + @Test + void symlinkEscapingTheWorkingDirectoryIsNotReturnedAsADimensionPath(@TempDir Path root, @TempDir Path outside) + throws IOException { + createRegionDir(root.resolve("world")); + Path secretRegion = outside.resolve("secret").resolve("region"); + Files.createDirectories(secretRegion); + Path netherLink = root.resolve("world").resolve("DIM-1"); + try { + Files.createSymbolicLink(netherLink, outside.resolve("secret")); + } catch (IOException | UnsupportedOperationException e) { + Assumptions.abort("Symbolic links are not supported on this filesystem: " + e.getMessage()); + return; + } + + WorldLayout layout = LayoutDetector.detect(root, "world", null); + + assertEquals("vanilla", layout.kind()); + assertEquals( + root.resolve("world").resolve("region"), + layout.dimensions().get("overworld")); + assertFalse( + layout.dimensions().containsKey("the_nether"), + "a dimension reached only through a symlink escaping the root must not be reported"); + } + + @Test + void symlinkInPlaceOfARegionDirectoryIsRejected(@TempDir Path root, @TempDir Path outside) throws IOException { + Files.createDirectories(root.resolve("world")); + try { + Files.createSymbolicLink(root.resolve("world").resolve("region"), outside); + } catch (IOException | UnsupportedOperationException e) { + Assumptions.abort("Symbolic links are not supported on this filesystem: " + e.getMessage()); + return; + } + + assertThrows(LayoutDetectionException.class, () -> LayoutDetector.detect(root, "world", null)); + } + + private static void createRegionDir(Path dimensionDir) throws IOException { + Files.createDirectories(dimensionDir.resolve("region")); + } +} diff --git a/ingest/src/test/java/net/onelitefeather/apus/ingest/S3ClientTest.java b/ingest/src/test/java/net/onelitefeather/apus/ingest/S3ClientTest.java new file mode 100644 index 0000000..149557a --- /dev/null +++ b/ingest/src/test/java/net/onelitefeather/apus/ingest/S3ClientTest.java @@ -0,0 +1,70 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; + +class S3ClientTest { + + /** A hand-written double for the AWS SDK's own client: only {@code putObject} is overridden. */ + private static final class RecordingSdkS3Client implements software.amazon.awssdk.services.s3.S3Client { + private PutObjectRequest lastRequest; + private byte[] lastBody; + + @Override + public PutObjectResponse putObject(PutObjectRequest putObjectRequest, RequestBody requestBody) { + this.lastRequest = putObjectRequest; + try { + this.lastBody = requestBody.contentStreamProvider().newStream().readAllBytes(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return PutObjectResponse.builder().build(); + } + + @Override + public String serviceName() { + return "s3"; + } + + @Override + public void close() {} + } + + @Test + void wrappingDelegatesBucketKeyAndContentToTheSdkClient() { + RecordingSdkS3Client delegate = new RecordingSdkS3Client(); + S3Client facade = S3Client.wrapping(delegate); + byte[] content = "region bytes".getBytes(StandardCharsets.UTF_8); + + facade.putObject("my-bucket", "acme/world/v1/manifest.json", content); + + assertEquals("my-bucket", delegate.lastRequest.bucket()); + assertEquals("acme/world/v1/manifest.json", delegate.lastRequest.key()); + assertArrayEquals(content, delegate.lastBody); + } +} diff --git a/ingest/src/test/java/net/onelitefeather/apus/ingest/ThrottledProgressSinkTest.java b/ingest/src/test/java/net/onelitefeather/apus/ingest/ThrottledProgressSinkTest.java new file mode 100644 index 0000000..1850676 --- /dev/null +++ b/ingest/src/test/java/net/onelitefeather/apus/ingest/ThrottledProgressSinkTest.java @@ -0,0 +1,94 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.Iterator; +import java.util.List; +import org.junit.jupiter.api.Test; + +class ThrottledProgressSinkTest { + + @Test + void printsOnTheFirstUpdateThenSuppressesUntilTheIntervalElapses() { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + // One fixed instant per call, so every update looks simultaneous -- none but the first + // and the (necessarily final, bytesDone == bytesTotal) update should print. + Instant frozen = Instant.parse("2026-08-08T00:00:00Z"); + ThrottledProgressSink sink = new ThrottledProgressSink(Duration.ofSeconds(10), () -> frozen, printStream(buffer)); + + sink.update(10, 100); + sink.update(20, 100); + sink.update(30, 100); + + List lines = lines(buffer); + assertEquals(1, lines.size(), "only the first call may print while the clock stands still: " + lines); + assertTrue(lines.get(0).contains("10/100")); + } + + @Test + void printsAgainOnceTheIntervalHasElapsed() { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + Iterator ticks = List.of( + Instant.parse("2026-08-08T00:00:00Z"), // first update: always prints + Instant.parse("2026-08-08T00:00:03Z"), // +3s, interval is 10s: suppressed + Instant.parse("2026-08-08T00:00:11Z")) // +11s since last print: prints + .iterator(); + ThrottledProgressSink sink = new ThrottledProgressSink(Duration.ofSeconds(10), ticks::next, printStream(buffer)); + + sink.update(10, 1000); + sink.update(20, 1000); + sink.update(30, 1000); + + List lines = lines(buffer); + assertEquals(2, lines.size(), "first update and the one 11s later, not the one in between: " + lines); + assertTrue(lines.get(0).contains("10/1000")); + assertTrue(lines.get(1).contains("30/1000")); + } + + @Test + void alwaysPrintsTheFinalUpdateEvenWithinTheThrottleWindow() { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + Instant frozen = Instant.parse("2026-08-08T00:00:00Z"); + ThrottledProgressSink sink = new ThrottledProgressSink(Duration.ofMinutes(5), () -> frozen, printStream(buffer)); + + sink.update(50, 100); + sink.update(100, 100); // final, despite zero elapsed time on the frozen clock + + List lines = lines(buffer); + assertEquals(2, lines.size()); + assertTrue(lines.get(1).contains("100.0%")); + assertTrue(lines.get(1).contains("100/100")); + } + + private static PrintStream printStream(ByteArrayOutputStream buffer) { + return new PrintStream(buffer, true, StandardCharsets.UTF_8); + } + + private static List lines(ByteArrayOutputStream buffer) { + String text = buffer.toString(StandardCharsets.UTF_8); + return text.isEmpty() ? List.of() : List.of(text.strip().split("\\R")); + } +} diff --git a/ingest/src/test/java/net/onelitefeather/apus/ingest/connector/ArchivesTest.java b/ingest/src/test/java/net/onelitefeather/apus/ingest/connector/ArchivesTest.java new file mode 100644 index 0000000..99e1c56 --- /dev/null +++ b/ingest/src/test/java/net/onelitefeather/apus/ingest/connector/ArchivesTest.java @@ -0,0 +1,134 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest.connector; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Proves {@link Archives#extractTar} resists the same class of attack that {@link + * net.onelitefeather.apus.ingest.LayoutDetector} was found and hardened against (see that + * class's Javadoc): a tar archive is untrusted input (a Pterodactyl backup, or an object fetched + * from S3), so an entry name must never be able to write outside the target directory, and a + * symlink-typed entry must never be materialised as a real filesystem symlink. + * + *

Both protections already existed in {@link Archives} before this test was written -- + * {@code resolveSafely} normalises and contains every entry path, and neither {@code extractTar} + * nor {@link TarStreamReader} ever calls {@code Files.createSymbolicLink}, so a {@code typeflag + * '2'} entry is written as an inert regular file rather than a link. These tests exist as + * regression coverage for that behaviour, not as a fix for a bug found here. + */ +class ArchivesTest { + + @Test + void extractTarRejectsAnEntryThatEscapesTheTargetDirectoryViaDotDotSegments(@TempDir Path targetDir) + throws IOException { + byte[] tar = new TestTarBuilder().addFile("../evil.txt", "pwned").toTarBytes(); + + IOException thrown = assertThrows( + IOException.class, + () -> Archives.extractTar(new ByteArrayInputStream(tar), targetDir, entryName -> true)); + assertTrue( + thrown.getMessage().contains("escapes target directory"), + "expected a path-escape message, got: " + thrown.getMessage()); + + assertFalse( + Files.exists(targetDir.getParent().resolve("evil.txt")), + "the '..' entry must not have landed a file next to the target directory"); + } + + @Test + void extractTarDoesNotMaterializeATarSymlinkEntryAsARealSymlink(@TempDir Path targetDir) throws IOException { + byte[] tar = new TestTarBuilder().addSymlink("escape-link", "/etc/passwd").toTarBytes(); + + Archives.extractTar(new ByteArrayInputStream(tar), targetDir, entryName -> true); + + Path written = targetDir.resolve("escape-link"); + assertTrue(Files.exists(written), "the entry is still materialised, just not as a link"); + assertFalse( + Files.isSymbolicLink(written), + "a tar symlink entry must never become a real filesystem symlink pointing outside the target"); + assertEquals(0L, Files.size(written), "the reader does not (yet) resolve link targets as content"); + } + + // --- S2: bounded extraction (zip-bomb protection) -------------------------------------- + + @Test + void extractTarAbortsOnceTheTotalSizeLimitIsExceeded(@TempDir Path targetDir) throws IOException { + byte[] tar = new TestTarBuilder() + .addFile("world/region/r.0.0.mca", "0123456789") // 10 bytes + .toTarBytes(); + + Archives.Limits limits = new Archives.Limits(5, Long.MAX_VALUE); // smaller than the one entry + + IOException thrown = assertThrows( + IOException.class, + () -> Archives.extractTar(new ByteArrayInputStream(tar), targetDir, entryName -> true, limits)); + assertTrue( + thrown.getMessage().contains("total size limit"), + "expected a total-size-limit message, got: " + thrown.getMessage()); + } + + @Test + void extractTarAbortsOnceTheEntryCountLimitIsExceeded(@TempDir Path targetDir) throws IOException { + byte[] tar = new TestTarBuilder() + .addFile("world/region/r.0.0.mca", "a") + .addFile("world/region/r.0.1.mca", "b") + .addFile("world/region/r.0.2.mca", "c") + .toTarBytes(); + + Archives.Limits limits = new Archives.Limits(Long.MAX_VALUE, 2); + + IOException thrown = assertThrows( + IOException.class, + () -> Archives.extractTar(new ByteArrayInputStream(tar), targetDir, entryName -> true, limits)); + assertTrue( + thrown.getMessage().contains("entry limit"), "expected an entry-limit message, got: " + thrown.getMessage()); + } + + @Test + void extractTarWithinLimitsSucceedsNormally(@TempDir Path targetDir) throws IOException { + byte[] tar = new TestTarBuilder().addFile("world/region/r.0.0.mca", "content").toTarBytes(); + + Archives.extractTar(new ByteArrayInputStream(tar), targetDir, entryName -> true, new Archives.Limits(1024, 10)); + + assertTrue(Files.exists(targetDir.resolve("world/region/r.0.0.mca"))); + } + + @Test + void limitsFromReadsConfiguredValuesAndDefaultsToUnboundedWhenAbsent() { + Archives.Limits configured = Archives.limitsFrom( + Map.of(Archives.CONFIG_MAX_TOTAL_BYTES, "42", Archives.CONFIG_MAX_ENTRIES, "7")); + assertEquals(42L, configured.maxTotalBytes()); + assertEquals(7L, configured.maxEntries()); + + Archives.Limits defaulted = Archives.limitsFrom(Map.of()); + assertEquals(Long.MAX_VALUE, defaulted.maxTotalBytes()); + assertEquals(Long.MAX_VALUE, defaulted.maxEntries()); + } +} diff --git a/ingest/src/test/java/net/onelitefeather/apus/ingest/connector/PterodactylConnectorTest.java b/ingest/src/test/java/net/onelitefeather/apus/ingest/connector/PterodactylConnectorTest.java new file mode 100644 index 0000000..d952ead --- /dev/null +++ b/ingest/src/test/java/net/onelitefeather/apus/ingest/connector/PterodactylConnectorTest.java @@ -0,0 +1,323 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest.connector; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.reflect.Field; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.http.HttpClient; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Exercises {@link PterodactylConnector} against a local HTTP stub that reproduces the Pterodactyl + * Client API's documented backup endpoints and response shapes (see the connector's class Javadoc + * for exactly which parts are sourced from the panel's own code vs. community docs). No real panel + * is contacted, and the stub binds to the loopback address only. + */ +class PterodactylConnectorTest { + + private static final String SERVER_ID = "srv-1"; + private static final String API_KEY = "ptlc_test_key"; + private static final String SUCCESSFUL_BACKUP_UUID = "11111111-1111-1111-1111-111111111111"; + + private HttpServer server; + + @AfterEach + void tearDown() { + if (server != null) { + server.stop(0); + } + } + + @Test + void discoverListsOnlySuccessfulBackupsAndSendsTheBearerToken() throws IOException { + startServer(null); + + List versions = new PterodactylConnector().discover(baseConfig()); + + assertEquals(1, versions.size(), "the still-running backup must be filtered out"); + SourceVersion version = versions.get(0); + assertEquals(SUCCESSFUL_BACKUP_UUID, version.id()); + assertEquals("daily-backup", version.label()); + assertEquals(123456L, version.sizeBytes()); + assertEquals(Instant.parse("2026-08-01T10:00:00Z"), version.createdAt()); + } + + /** + * The core requirement this task exists to prove: the backup is a tar.gz of the *entire* + * server (plugins, configs, worlds all mixed together), gzip is not seekable, so the stream is + * walked exactly once and only the configured world paths are written to disk -- nothing else + * from the archive ever lands in the work directory. + */ + @Test + void fetchExtractsOnlyWorldPathsFromAMixedServerBackupInOnePass(@TempDir Path workDir) throws IOException { + byte[] mixedBackup = new TestTarBuilder() + .addFile("server.properties", "level-name=world") + .addDirectory("plugins") + .addFile("plugins/Essentials/config.yml", "locale: en") + .addFile("plugins/Essentials/userdata/uuid.yml", "balance: 100") + .addDirectory("logs") + .addFile("logs/latest.log", "[INFO] server started") + .addFile("world/level.dat", "overworld-level-data") + .addFile("world/region/r.0.0.mca", "overworld-region-data") + .addFile("world_nether/DIM-1/region/r.0.0.mca", "nether-region-data") + .toGzippedTarBytes(); + startServer(mixedBackup); + + Map config = baseConfig(); + config.put(PterodactylConnector.CONFIG_WORLD_PATHS, "world,world_nether"); + SourceVersion version = new SourceVersion(SUCCESSFUL_BACKUP_UUID, "daily-backup", Instant.now(), mixedBackup.length); + + new PterodactylConnector().fetch(config, version, workDir); + + assertEquals("overworld-level-data", Files.readString(workDir.resolve("world/level.dat"))); + assertEquals("overworld-region-data", Files.readString(workDir.resolve("world/region/r.0.0.mca"))); + assertEquals( + "nether-region-data", Files.readString(workDir.resolve("world_nether/DIM-1/region/r.0.0.mca"))); + + assertFalse(Files.exists(workDir.resolve("server.properties")), "server.properties is not part of a world"); + assertFalse(Files.exists(workDir.resolve("plugins")), "plugin data is not part of a world"); + assertFalse(Files.exists(workDir.resolve("logs")), "log files are not part of a world"); + } + + /** + * F1: if the panel's response envelope ever stops matching {@code + * {"object":"list","data":[...]}} -- a different API version, a proxy error page served with + * a 2xx status, whatever -- {@code discover()} must fail loudly rather than silently reading + * an empty backup list out of the mismatched shape (which {@code WorldSourceReconciler} would + * report as a perfectly healthy "no versions available at the source yet"). + */ + @Test + void discoverRejectsAResponseThatDoesNotMatchTheExpectedListEnvelope() throws IOException { + server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + server.createContext("/api/client/servers/" + SERVER_ID + "/backups", exchange -> { + requireBearerToken(exchange); + // Structurally valid JSON, but not the {object:"list", data:[...]} envelope this + // connector's whole parsing logic assumes. + respondJson(exchange, 200, "{\"unexpected\":\"shape\"}"); + }); + server.start(); + + IllegalStateException e = + assertThrows(IllegalStateException.class, () -> new PterodactylConnector().discover(baseConfig())); + assertTrue(e.getMessage().contains("expected"), "must explain what shape was expected, not just fail blankly"); + } + + /** + * S1: a failure downloading the backup archive must never embed the signed URL itself in its + * message -- it is a short-lived but fully-privileged credential for the entire backup, and + * exception messages end up on stderr, i.e. in log aggregation (see {@code IngestMain}). + */ + @Test + void fetchDoesNotLeakTheSignedDownloadUrlWhenTheDownloadFails(@TempDir Path workDir) throws IOException { + int deadPort = unusedPort(); + String secretToken = "super-secret-signed-token-should-never-appear-in-logs"; + + server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + server.createContext("/api/client/servers/" + SERVER_ID + "/backups", exchange -> { + requireBearerToken(exchange); + respondJson(exchange, 200, backupsListResponse()); + }); + server.createContext( + "/api/client/servers/" + SERVER_ID + "/backups/" + SUCCESSFUL_BACKUP_UUID + "/download", exchange -> { + requireBearerToken(exchange); + // Points at a closed local port: the GET against it fails with a connection + // refused IOException, exercising fetch()'s catch(IOException) branch. + String deadUrl = "http://127.0.0.1:" + deadPort + "/archive.tar.gz?token=" + secretToken; + respondJson(exchange, 200, "{\"object\":\"signed_url\",\"attributes\":{\"url\":\"" + deadUrl + "\"}}"); + }); + server.start(); + + Map config = baseConfig(); + config.put(PterodactylConnector.CONFIG_WORLD_PATHS, "world"); + SourceVersion version = new SourceVersion(SUCCESSFUL_BACKUP_UUID, "daily-backup", Instant.now(), 0); + + UncheckedIOException e = assertThrows( + UncheckedIOException.class, () -> new PterodactylConnector().fetch(config, version, workDir)); + + assertFalse( + e.getMessage().contains(secretToken), + "the signed URL (and its token) must never appear in an exception message, got: " + e.getMessage()); + assertTrue( + e.getMessage().contains("127.0.0.1"), "the host alone is still useful for diagnosing the failure"); + } + + /** + * B1: {@code discover()}/{@code fetch()} run inside a JOSDK worker shared across every + * reconciler (see the class Javadoc); a panel that accepts a connection and never responds + * must not be able to hang that worker forever. + */ + @Test + void theDefaultHttpClientHasAConnectTimeoutConfigured() throws ReflectiveOperationException { + PterodactylConnector connector = new PterodactylConnector(); + + Field field = PterodactylConnector.class.getDeclaredField("httpClient"); + field.setAccessible(true); + HttpClient client = (HttpClient) field.get(connector); + + assertTrue( + client.connectTimeout().isPresent(), + "the default client must bound how long connecting to an unresponsive panel can take"); + } + + private static int unusedPort() throws IOException { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } + + private Map baseConfig() { + Map config = new HashMap<>(); + config.put(PterodactylConnector.CONFIG_PANEL_URL, "http://127.0.0.1:" + server.getAddress().getPort()); + config.put(PterodactylConnector.CONFIG_SERVER_ID, SERVER_ID); + config.put(PterodactylConnector.CONFIG_API_KEY, API_KEY); + return config; + } + + /** + * Starts the panel + download stub on the loopback address only. When {@code archiveBytes} is + * non-null, a third endpoint serves it as the signed-URL download target. + */ + private void startServer(byte[] archiveBytes) throws IOException { + server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + + server.createContext("/api/client/servers/" + SERVER_ID + "/backups", exchange -> { + if (!("/api/client/servers/" + SERVER_ID + "/backups").equals(exchange.getRequestURI().getPath())) { + exchange.sendResponseHeaders(404, -1); + exchange.close(); + return; + } + requireBearerToken(exchange); + respondJson(exchange, 200, backupsListResponse()); + }); + + server.createContext( + "/api/client/servers/" + SERVER_ID + "/backups/" + SUCCESSFUL_BACKUP_UUID + "/download", exchange -> { + requireBearerToken(exchange); + String archiveUrl = "http://127.0.0.1:" + server.getAddress().getPort() + "/archive.tar.gz"; + respondJson( + exchange, + 200, + "{\"object\":\"signed_url\",\"attributes\":{\"url\":\"" + archiveUrl + "\"}}"); + }); + + if (archiveBytes != null) { + server.createContext("/archive.tar.gz", exchange -> { + exchange.getResponseHeaders().add("Content-Type", "application/gzip"); + exchange.sendResponseHeaders(200, archiveBytes.length); + try (var out = exchange.getResponseBody()) { + out.write(archiveBytes); + } + }); + } + + server.start(); + } + + private static void requireBearerToken(HttpExchange exchange) throws IOException { + String authorization = exchange.getRequestHeaders().getFirst("Authorization"); + if (!("Bearer " + API_KEY).equals(authorization)) { + exchange.sendResponseHeaders(401, -1); + exchange.close(); + throw new IOException("unauthorized stub request, aborting handler"); + } + } + + private static void respondJson(HttpExchange exchange, int status, String body) throws IOException { + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, bytes.length); + try (var out = exchange.getResponseBody()) { + out.write(bytes); + } + } + + /** + * Response shape verified against the Pterodactyl panel source directly (see {@link + * PterodactylConnector}'s class Javadoc): {@code BackupTransformer} attribute names, and + * {@code BackupController::index}'s pagination wrapper as documented by the panel's API docs + * mirrors. + */ + private static String backupsListResponse() { + return """ + { + "object": "list", + "data": [ + { + "object": "backup", + "attributes": { + "uuid": "%s", + "is_successful": true, + "is_locked": false, + "name": "daily-backup", + "ignored_files": [], + "checksum": "sha256:aaaabbbbcccc", + "bytes": 123456, + "created_at": "2026-08-01T10:00:00+00:00", + "completed_at": "2026-08-01T10:05:00+00:00" + } + }, + { + "object": "backup", + "attributes": { + "uuid": "22222222-2222-2222-2222-222222222222", + "is_successful": false, + "is_locked": false, + "name": "still-running-backup", + "ignored_files": [], + "checksum": null, + "bytes": 0, + "created_at": "2026-08-02T10:00:00+00:00", + "completed_at": null + } + } + ], + "meta": { + "pagination": { + "total": 2, + "count": 2, + "per_page": 50, + "current_page": 1, + "total_pages": 1 + }, + "backup_count": 2 + } + } + """ + .formatted(SUCCESSFUL_BACKUP_UUID); + } +} diff --git a/ingest/src/test/java/net/onelitefeather/apus/ingest/connector/S3SourceConnectorTest.java b/ingest/src/test/java/net/onelitefeather/apus/ingest/connector/S3SourceConnectorTest.java new file mode 100644 index 0000000..cc572bb --- /dev/null +++ b/ingest/src/test/java/net/onelitefeather/apus/ingest/connector/S3SourceConnectorTest.java @@ -0,0 +1,199 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest.connector; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.SecureRandom; +import java.time.Instant; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.testcontainers.containers.MinIOContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; + +/** + * Exercises {@link S3SourceConnector} against a real MinIO instance -- listing/versioning + * semantics (delimiter-scoped listing, last-modified/size metadata) are exactly the kind of + * behaviour a hand-rolled stub would get subtly wrong. + * + *

MinIO is started via Testcontainers ({@link MinIOContainer}), the same mechanism the + * {@code operator} and {@code runner} modules already use for their own container-based tests. + * Testcontainers reaps the container even if a test crashes, which the previous {@code + * ProcessBuilder}-driven {@code docker run}/{@code docker stop} pairing here could not guarantee. + */ +@Testcontainers +class S3SourceConnectorTest { + + private static final String BUCKET = "worlds"; + + // Generated fresh per test run rather than pinned to a fixed literal, so nothing checked + // into source ever looks like a real credential. Lengths follow MinIO's own + // accessKeyMinLen/secretKeyMinLen (3 / 8 characters) with generous headroom. + private static final String ACCESS_KEY = randomAlphanumeric(20); + private static final String SECRET_KEY = randomAlphanumeric(40); + + @Container + private static final MinIOContainer MINIO = + new MinIOContainer(DockerImageName.parse("minio/minio:RELEASE.2024-11-07T00-52-20Z")) + .withUserName(ACCESS_KEY) + .withPassword(SECRET_KEY); + + private static S3Client sharedClient; + + /** Generates a random alphanumeric string of {@code length} characters. */ + private static String randomAlphanumeric(int length) { + String alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + SecureRandom random = new SecureRandom(); + StringBuilder value = new StringBuilder(length); + for (int i = 0; i < length; i++) { + value.append(alphabet.charAt(random.nextInt(alphabet.length()))); + } + return value.toString(); + } + + @BeforeAll + static void createClientAndBucket() { + sharedClient = S3Client.builder() + .endpointOverride(URI.create(MINIO.getS3URL())) + .region(Region.US_EAST_1) + .credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create(ACCESS_KEY, SECRET_KEY))) + .forcePathStyle(true) + .build(); + sharedClient.createBucket(CreateBucketRequest.builder().bucket(BUCKET).build()); + } + + @AfterAll + static void closeClient() { + sharedClient.close(); + } + + private final S3SourceConnector connector = new S3SourceConnector(); + + @Test + void discoverReportsOneVersionPerObjectDirectlyUnderThePrefix() { + String prefix = "discover-test/"; + putObject(prefix + "2026-08-01T00-00-00Z.zip", "zip-bytes-placeholder"); + putObject(prefix + "2026-08-02T00-00-00Z.tar.gz", "tar-gz-bytes-placeholder"); + // an object one level deeper must not be mistaken for a version of its own + putObject(prefix + "2026-08-02T00-00-00Z.tar.gz/unexpected-nested-object", "noise"); + + List versions = connector.discover(sharedClient, config(prefix)); + + Set ids = versions.stream().map(SourceVersion::id).collect(Collectors.toSet()); + assertEquals(Set.of("2026-08-01T00-00-00Z.zip", "2026-08-02T00-00-00Z.tar.gz"), ids); + } + + @Test + void fetchOfAZipKeyExtractsItsEntriesIntoTheWorkDirectory(@TempDir Path workDir) throws IOException { + String prefix = "zip-test/"; + byte[] zip = buildZip(Map.of( + "level.dat", "level-data", + "region/r.0.0.mca", "region-data")); + putObject(prefix + "v1.zip", zip); + + connector.fetch(sharedClient, config(prefix), new SourceVersion("v1.zip", "v1.zip", Instant.now(), zip.length), workDir); + + assertEquals("level-data", Files.readString(workDir.resolve("level.dat"))); + assertEquals("region-data", Files.readString(workDir.resolve("region/r.0.0.mca"))); + } + + @Test + void fetchOfATarGzKeyExtractsItsEntriesIntoTheWorkDirectory(@TempDir Path workDir) throws IOException { + String prefix = "targz-test/"; + byte[] tarGz = new TestTarBuilder() + .addFile("level.dat", "level-data") + .addFile("region/r.0.0.mca", "region-data") + .toGzippedTarBytes(); + putObject(prefix + "v2.tar.gz", tarGz); + + connector.fetch( + sharedClient, config(prefix), new SourceVersion("v2.tar.gz", "v2.tar.gz", Instant.now(), tarGz.length), workDir); + + assertEquals("level-data", Files.readString(workDir.resolve("level.dat"))); + assertEquals("region-data", Files.readString(workDir.resolve("region/r.0.0.mca"))); + } + + @Test + void fetchOfAPlainKeyWritesItAsASingleRawFile(@TempDir Path workDir) throws IOException { + String prefix = "raw-test/"; + putObject(prefix + "raw-dump.bin", "not-an-archive"); + + connector.fetch( + sharedClient, config(prefix), new SourceVersion("raw-dump.bin", "raw-dump.bin", Instant.now(), 14), workDir); + + assertTrue(Files.exists(workDir.resolve("raw-dump.bin"))); + assertEquals("not-an-archive", Files.readString(workDir.resolve("raw-dump.bin"))); + } + + private Map config(String prefix) { + Map config = new HashMap<>(); + config.put(S3SourceConnector.CONFIG_BUCKET, BUCKET); + config.put(S3SourceConnector.CONFIG_PREFIX, prefix); + return config; + } + + private void putObject(String key, String content) { + putObject(key, content.getBytes(StandardCharsets.UTF_8)); + } + + private void putObject(String key, byte[] content) { + sharedClient.putObject( + PutObjectRequest.builder().bucket(BUCKET).key(key).build(), RequestBody.fromBytes(content)); + } + + private static byte[] buildZip(Map entries) throws IOException { + var buffer = new ByteArrayOutputStream(); + try (var zip = new ZipOutputStream(buffer)) { + for (var entry : entries.entrySet().stream() + .sorted(Map.Entry.comparingByKey(Comparator.naturalOrder())) + .toList()) { + zip.putNextEntry(new ZipEntry(entry.getKey())); + zip.write(entry.getValue().getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + } + } + return buffer.toByteArray(); + } +} diff --git a/ingest/src/test/java/net/onelitefeather/apus/ingest/connector/TarStreamReaderTest.java b/ingest/src/test/java/net/onelitefeather/apus/ingest/connector/TarStreamReaderTest.java new file mode 100644 index 0000000..38db421 --- /dev/null +++ b/ingest/src/test/java/net/onelitefeather/apus/ingest/connector/TarStreamReaderTest.java @@ -0,0 +1,145 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest.connector; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.api.Test; + +/** + * Direct tests of the hand-rolled tar reader, independent of any HTTP or S3 plumbing, since it is + * the one piece of genuinely novel parsing logic in this package. + */ +class TarStreamReaderTest { + + @Test + void readsARegularFileEntryBackByteForByte() throws IOException { + byte[] content = "hello world".getBytes(StandardCharsets.UTF_8); + byte[] tar = new TestTarBuilder().addFile("greeting.txt", content).toTarBytes(); + + try (TarStreamReader reader = new TarStreamReader(new ByteArrayInputStream(tar))) { + TarStreamReader.Entry entry = reader.nextEntry(); + assertEquals("greeting.txt", entry.name()); + assertFalse(entry.directory()); + assertEquals(content.length, entry.size()); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + reader.transferTo(out); + assertArrayEquals(content, out.toByteArray()); + + assertNull(reader.nextEntry(), "the end-of-archive marker must surface as null"); + } + } + + @Test + void reportsDirectoryEntriesAsDirectoriesWithNoContentToRead() throws IOException { + byte[] tar = new TestTarBuilder().addDirectory("world").toTarBytes(); + + try (TarStreamReader reader = new TarStreamReader(new ByteArrayInputStream(tar))) { + TarStreamReader.Entry entry = reader.nextEntry(); + assertEquals("world/", entry.name()); + assertTrue(entry.directory()); + assertEquals(0, entry.size()); + } + } + + @Test + void readsMultipleEntriesInOrderEvenWhenAnEarlierEntrysContentIsNeverConsumed() throws IOException { + byte[] first = "first-file-content".getBytes(StandardCharsets.UTF_8); + byte[] second = "second-file-content".getBytes(StandardCharsets.UTF_8); + byte[] tar = new TestTarBuilder() + .addFile("a.txt", first) + .addFile("b.txt", second) + .toTarBytes(); + + try (TarStreamReader reader = new TarStreamReader(new ByteArrayInputStream(tar))) { + TarStreamReader.Entry entryA = reader.nextEntry(); + assertEquals("a.txt", entryA.name()); + // Deliberately skip transferTo() here -- nextEntry() must still land correctly on + // "b.txt" by skipping the unread content and padding itself. This is exactly the + // "walk once, skip what you don't want" behaviour the Pterodactyl connector relies on + // to avoid ever landing the full archive on disk. + + TarStreamReader.Entry entryB = reader.nextEntry(); + assertEquals("b.txt", entryB.name()); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + reader.transferTo(out); + assertArrayEquals(second, out.toByteArray()); + + assertNull(reader.nextEntry()); + } + } + + @Test + void reconstructsAGnuLongNameEntryLongerThanTheHundredByteHeaderField() throws IOException { + String longName = + "plugins/SomeReallyLongPluginNameThatDefinitelyExceedsTheClassicHundredByteUstarNameFieldLimitByAWideMargin/config.yml"; + assertTrue(longName.length() > 100, "test setup must actually exercise the long-name path"); + byte[] content = "key: value".getBytes(StandardCharsets.UTF_8); + byte[] tar = new TestTarBuilder().addFileWithLongName(longName, content).toTarBytes(); + + try (TarStreamReader reader = new TarStreamReader(new ByteArrayInputStream(tar))) { + TarStreamReader.Entry entry = reader.nextEntry(); + assertEquals(longName, entry.name()); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + reader.transferTo(out); + assertArrayEquals(content, out.toByteArray()); + } + } + + @Test + void reconstructsAPaxPathOverrideEntryLongerThanTheHundredByteHeaderField() throws IOException { + // The PAX case (typeflag 'x') is the sibling of the GNU long-name case above + // (typeflag 'L') exercised by reconstructsAGnuLongNameEntryLongerThanTheHundredByteField + // -- both extend an entry's path past the classic 100-byte ustar name field, and a + // Pterodactyl backup of a server that has grown deep plugin/world directory trees + // routinely contains both. Only the GNU case had a dedicated test before this one. + String longName = + "plugins/AnotherReallyLongPluginNameThatDefinitelyExceedsTheClassicHundredByteUstarNameFieldLimitByAWideMargin/config.yml"; + assertTrue(longName.length() > 100, "test setup must actually exercise the PAX path-override path"); + byte[] content = "key: value".getBytes(StandardCharsets.UTF_8); + byte[] tar = new TestTarBuilder().addFileWithPaxPathOverride(longName, content).toTarBytes(); + + try (TarStreamReader reader = new TarStreamReader(new ByteArrayInputStream(tar))) { + TarStreamReader.Entry entry = reader.nextEntry(); + assertEquals(longName, entry.name()); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + reader.transferTo(out); + assertArrayEquals(content, out.toByteArray()); + + assertNull(reader.nextEntry()); + } + } + + @Test + void anEmptyArchiveYieldsNoEntries() throws IOException { + byte[] tar = new TestTarBuilder().toTarBytes(); + + try (TarStreamReader reader = new TarStreamReader(new ByteArrayInputStream(tar))) { + assertNull(reader.nextEntry()); + } + } +} diff --git a/ingest/src/test/java/net/onelitefeather/apus/ingest/connector/TestTarBuilder.java b/ingest/src/test/java/net/onelitefeather/apus/ingest/connector/TestTarBuilder.java new file mode 100644 index 0000000..36a83b2 --- /dev/null +++ b/ingest/src/test/java/net/onelitefeather/apus/ingest/connector/TestTarBuilder.java @@ -0,0 +1,173 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.ingest.connector; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.zip.GZIPOutputStream; + +/** + * Hand-builds minimal ustar/GNU-tar byte streams for tests, since the module has no archive + * library dependency to build real ones with (see {@link TarStreamReader}'s Javadoc for why). + */ +final class TestTarBuilder { + + private static final int BLOCK_SIZE = 512; + + private final ByteArrayOutputStream body = new ByteArrayOutputStream(); + + TestTarBuilder addFile(String name, byte[] content) { + writeHeader(name, content.length, '0'); + body.writeBytes(content); + writePadding(content.length); + return this; + } + + TestTarBuilder addFile(String name, String content) { + return addFile(name, content.getBytes(StandardCharsets.UTF_8)); + } + + TestTarBuilder addDirectory(String name) { + String dirName = name.endsWith("/") ? name : name + "/"; + writeHeader(dirName, 0, '5'); + return this; + } + + /** Writes a GNU long-name entry pair: an {@code 'L'} header carrying the real name, then the file. */ + TestTarBuilder addFileWithLongName(String name, byte[] content) { + byte[] nameBytes = (name + "\0").getBytes(StandardCharsets.UTF_8); + writeHeader("", nameBytes.length, 'L'); + body.writeBytes(nameBytes); + writePadding(nameBytes.length); + + // The following regular-file header's own 100-byte name field is irrelevant once a + // preceding 'L' entry is present -- real GNU tar still fills something plausible in + // there, so this mirrors that instead of leaving it blank. + String truncated = name.length() > 100 ? name.substring(0, 100) : name; + writeHeader(truncated, content.length, '0'); + body.writeBytes(content); + writePadding(content.length); + return this; + } + + /** Writes a PAX extended-header entry pair: an {@code 'x'} header carrying a {@code path} record, then the file. */ + TestTarBuilder addFileWithPaxPathOverride(String name, byte[] content) { + byte[] paxBody = paxRecord("path", name).getBytes(StandardCharsets.UTF_8); + writeHeader("PaxHeaders.0/pax-entry", paxBody.length, 'x'); + body.writeBytes(paxBody); + writePadding(paxBody.length); + + // Same as addFileWithLongName: the following header's own name field is irrelevant once + // a preceding 'x' entry supplies the real path. + String truncated = name.length() > 100 ? name.substring(0, 100) : name; + writeHeader(truncated, content.length, '0'); + body.writeBytes(content); + writePadding(content.length); + return this; + } + + /** Writes a symlink entry ({@code typeflag '2'}) with no content, only a link target. */ + TestTarBuilder addSymlink(String name, String linkTarget) { + writeHeader(name, 0, '2', linkTarget); + return this; + } + + /** + * Builds one PAX extended-header record: {@code " =\n"}, where {@code + * } is the record's own total byte length including the length prefix itself (per + * the PAX format's self-describing, length-prefixed record layout). + */ + private static String paxRecord(String key, String value) { + String keyValue = key + "=" + value + "\n"; + int length = keyValue.length() + 2; + while (String.valueOf(length).length() + 1 + keyValue.length() != length) { + length = String.valueOf(length).length() + 1 + keyValue.length(); + } + return length + " " + keyValue; + } + + byte[] toTarBytes() { + ByteArrayOutputStream full = new ByteArrayOutputStream(); + full.writeBytes(body.toByteArray()); + full.writeBytes(new byte[BLOCK_SIZE * 2]); // two zero blocks mark end-of-archive + return full.toByteArray(); + } + + byte[] toGzippedTarBytes() throws IOException { + ByteArrayOutputStream gzipped = new ByteArrayOutputStream(); + try (GZIPOutputStream gzip = new GZIPOutputStream(gzipped)) { + gzip.write(toTarBytes()); + } + return gzipped.toByteArray(); + } + + private void writeHeader(String name, int size, char typeflag) { + writeHeader(name, size, typeflag, null); + } + + /** As {@link #writeHeader(String, int, char)}, but also fills in the linkname field (offset 157, 100 bytes) -- e.g. a symlink's target. */ + private void writeHeader(String name, int size, char typeflag, String linkName) { + byte[] header = new byte[BLOCK_SIZE]; + byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); + System.arraycopy(nameBytes, 0, header, 0, Math.min(nameBytes.length, 100)); + writeOctal(header, 100, 8, 0x1A4); // mode 0644 + writeOctal(header, 108, 8, 0); // uid + writeOctal(header, 116, 8, 0); // gid + writeOctal(header, 124, 12, size); + writeOctal(header, 136, 12, 0); // mtime + header[156] = (byte) typeflag; + if (linkName != null) { + byte[] linkNameBytes = linkName.getBytes(StandardCharsets.UTF_8); + System.arraycopy(linkNameBytes, 0, header, 157, Math.min(linkNameBytes.length, 100)); + } + byte[] magic = "ustar".getBytes(StandardCharsets.US_ASCII); + System.arraycopy(magic, 0, header, 257, magic.length); + header[263] = '0'; + header[264] = '0'; + + for (int i = 148; i < 156; i++) { + header[i] = ' '; + } + int checksum = 0; + for (byte b : header) { + checksum += (b & 0xFF); + } + byte[] checksumField = String.format("%06o\0 ", checksum).getBytes(StandardCharsets.US_ASCII); + System.arraycopy(checksumField, 0, header, 148, checksumField.length); + + body.writeBytes(header); + } + + private void writeOctal(byte[] header, int offset, int length, long value) { + String octal = Long.toOctalString(value); + StringBuilder padded = new StringBuilder(); + for (int i = 0; i < length - 1 - octal.length(); i++) { + padded.append('0'); + } + padded.append(octal); + byte[] bytes = padded.toString().getBytes(StandardCharsets.US_ASCII); + System.arraycopy(bytes, 0, header, offset, bytes.length); + header[offset + length - 1] = 0; + } + + private void writePadding(int size) { + int pad = (BLOCK_SIZE - (size % BLOCK_SIZE)) % BLOCK_SIZE; + body.writeBytes(new byte[pad]); + } +} diff --git a/operator/build.gradle.kts b/operator/build.gradle.kts index c26e95f..6a832ac 100644 --- a/operator/build.gradle.kts +++ b/operator/build.gradle.kts @@ -7,6 +7,25 @@ plugins { dependencies { implementation(libs.josdk) + // cron-utils -- see settings.gradle.kts for why. Backs CronSchedule, which + // WorldSourceReconciler (phase 2b, task 6) uses to evaluate spec.poll. + implementation(libs.cron.utils) + + // The ingest connectors (WorldSourceConnector/S3SourceConnector/PterodactylConnector) are + // reused directly by WorldSourceReconciler to run discover() on a schedule -- see + // ingest/README.md's "Design notes" section, which already documents this split: + // discover() (listing available versions) belongs to the reconciler, fetch() (pulling one + // specific version) belongs to the ingest Job this reconciler schedules. Depending on the + // module rather than duplicating the connector interface keeps exactly one implementation + // of each source type. + implementation(project(":ingest")) + + // AWS SDK v2 S3 client -- see settings.gradle.kts for why this over the MinIO Java client. + // Used by AwsBundleStore (phase 2b, task 6) to enforce WorldSource.spec.retention: listing + // and deleting older bundle versions in the destination bucket. + implementation(platform(libs.aws.sdk.bom)) + implementation(libs.aws.sdk.s3) + testImplementation(platform(libs.junit.bom)) testImplementation(libs.junit.jupiter) testRuntimeOnly(libs.junit.platform.launcher) diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/ApusOperator.java b/operator/src/main/java/net/onelitefeather/apus/operator/ApusOperator.java index da26b35..ddc025d 100644 --- a/operator/src/main/java/net/onelitefeather/apus/operator/ApusOperator.java +++ b/operator/src/main/java/net/onelitefeather/apus/operator/ApusOperator.java @@ -20,13 +20,15 @@ import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientBuilder; import io.javaoperatorsdk.operator.Operator; +import net.onelitefeather.apus.operator.ingest.WorldIngestReconciler; +import net.onelitefeather.apus.operator.ingest.WorldSourceReconciler; import net.onelitefeather.apus.operator.map.BlueMapMapReconciler; import net.onelitefeather.apus.operator.render.BlueMapRenderReconciler; import net.onelitefeather.apus.operator.tenant.TenantReconciler; /** * The operator's process entry point: builds a Kubernetes client and {@link OperatorConfig} from - * the environment, registers the three reconcilers against a single {@link Operator} instance, + * the environment, registers the five reconcilers against a single {@link Operator} instance, * and starts it. * *

There is no Micronaut (or any other framework) integration here on purpose -- the Java @@ -71,11 +73,12 @@ public static void main(String[] args) { return; } - System.out.println("[apus-operator] started, watching Tenant/BlueMapMap/BlueMapRender resources"); + System.out.println( + "[apus-operator] started, watching Tenant/BlueMapMap/BlueMapRender/WorldSource/WorldIngest resources"); } /** - * Registers all three reconcilers on {@code operator}. Extracted from {@link #main} so a + * Registers all five reconcilers on {@code operator}. Extracted from {@link #main} so a * test can exercise the wiring itself -- that every reconciler this operator ships is * actually registered -- against a mock {@link KubernetesClient} instead of a real cluster. */ @@ -83,6 +86,8 @@ static void registerReconcilers(Operator operator, KubernetesClient client, Oper operator.register(new TenantReconciler(client, config)); operator.register(new BlueMapMapReconciler(client, config)); operator.register(new BlueMapRenderReconciler(client, config)); + operator.register(new WorldSourceReconciler(client)); + operator.register(new WorldIngestReconciler(client, config)); } /** diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/OperatorConfig.java b/operator/src/main/java/net/onelitefeather/apus/operator/OperatorConfig.java index 829b40c..d80054b 100644 --- a/operator/src/main/java/net/onelitefeather/apus/operator/OperatorConfig.java +++ b/operator/src/main/java/net/onelitefeather/apus/operator/OperatorConfig.java @@ -21,27 +21,62 @@ /** * Site-specific settings the operator cannot derive from a Custom Resource: which Rook - * installation to talk to, and which runner image to schedule. Every reconciler in this - * module shares one instance, so it is modelled once here rather than re-created (and - * possibly re-defaulted differently) inside each of them. + * installation to talk to, which runner/ingest image to schedule, and where ingested world + * bundles are stored. Every reconciler in this module shares one instance, so it is modelled + * once here rather than re-created (and possibly re-defaulted differently) inside each of them. * * @param rookNamespace the namespace Rook's CRDs (ObjectBucketClaim, CephObjectStoreUser) live in * @param cephObjectStore the name of the Ceph object store to provision users/buckets against * @param bucketStorageClass the StorageClass used for {@code ObjectBucketClaim}s * @param runnerImage the container image running BlueMap renders + * @param ingestImage the container image running world ingest jobs (see {@code ingest/README.md}) + * @param bundleBucket the S3-compatible bucket every ingested world bundle is written to. + * Deliberately operator-wide rather than a {@code WorldSource}/{@code WorldIngest} spec + * field: neither phase 2b CRD carries a bundle-destination field of its own (only + * {@code WorldSourceSpec.s3}/{@code .pterodactyl}, which describe the raw data *source*), + * so this follows the same pattern {@link #runnerImage} already established for "a setting + * every tenant shares, not something a tenant configures per resource" + * @param bundleS3Endpoint the S3-compatible endpoint the bundle bucket above is reachable at + * @param bundleS3Region the region passed to the bundle destination's S3 client + * @param bundleCredentialsSecretName name of the {@code Secret} -- expected to exist in the + * same namespace as the {@code WorldSource}/{@code WorldIngest} being reconciled, mirroring + * how {@code RenderJobBuilder} is handed a bucket secret name already scoped to the map's + * namespace -- carrying the bundle bucket's {@code AWS_ACCESS_KEY_ID}/{@code + * AWS_SECRET_ACCESS_KEY} */ -public record OperatorConfig(String rookNamespace, String cephObjectStore, String bucketStorageClass, - String runnerImage) { +public record OperatorConfig( + String rookNamespace, + String cephObjectStore, + String bucketStorageClass, + String runnerImage, + String ingestImage, + String bundleBucket, + String bundleS3Endpoint, + String bundleS3Region, + String bundleCredentialsSecretName) { private static final String DEFAULT_ROOK_NAMESPACE = "rook-ceph-fr01"; private static final String DEFAULT_CEPH_OBJECT_STORE = "feather-s3"; private static final String DEFAULT_BUCKET_STORAGE_CLASS = "ceph-bucket-fr01"; private static final String DEFAULT_RUNNER_IMAGE = "apus/runner:dev"; + private static final String DEFAULT_INGEST_IMAGE = "apus/ingest:dev"; + private static final String DEFAULT_BUNDLE_BUCKET = "apus-bundles"; + private static final String DEFAULT_BUNDLE_S3_ENDPOINT = "http://rgw.rook-ceph-fr01.svc:80"; + private static final String DEFAULT_BUNDLE_S3_REGION = "us-east-1"; + private static final String DEFAULT_BUNDLE_CREDENTIALS_SECRET = "apus-bundle-credentials"; /** The feather-core cluster's actual values. */ public static OperatorConfig defaults() { return new OperatorConfig( - DEFAULT_ROOK_NAMESPACE, DEFAULT_CEPH_OBJECT_STORE, DEFAULT_BUCKET_STORAGE_CLASS, DEFAULT_RUNNER_IMAGE); + DEFAULT_ROOK_NAMESPACE, + DEFAULT_CEPH_OBJECT_STORE, + DEFAULT_BUCKET_STORAGE_CLASS, + DEFAULT_RUNNER_IMAGE, + DEFAULT_INGEST_IMAGE, + DEFAULT_BUNDLE_BUCKET, + DEFAULT_BUNDLE_S3_ENDPOINT, + DEFAULT_BUNDLE_S3_REGION, + DEFAULT_BUNDLE_CREDENTIALS_SECRET); } /** @@ -52,14 +87,21 @@ public static OperatorConfig defaults() { * directly so tests can supply a fake environment instead of mutating the real one. * *

Recognised variables: {@code APUS_ROOK_NAMESPACE}, {@code APUS_CEPH_OBJECT_STORE}, - * {@code APUS_BUCKET_STORAGE_CLASS}, {@code APUS_RUNNER_IMAGE}. + * {@code APUS_BUCKET_STORAGE_CLASS}, {@code APUS_RUNNER_IMAGE}, {@code APUS_INGEST_IMAGE}, + * {@code APUS_BUNDLE_BUCKET}, {@code APUS_BUNDLE_S3_ENDPOINT}, {@code + * APUS_BUNDLE_S3_REGION}, {@code APUS_BUNDLE_CREDENTIALS_SECRET}. */ public static OperatorConfig fromEnvironment(Function env) { return new OperatorConfig( valueOrDefault(env.apply("APUS_ROOK_NAMESPACE"), DEFAULT_ROOK_NAMESPACE), valueOrDefault(env.apply("APUS_CEPH_OBJECT_STORE"), DEFAULT_CEPH_OBJECT_STORE), valueOrDefault(env.apply("APUS_BUCKET_STORAGE_CLASS"), DEFAULT_BUCKET_STORAGE_CLASS), - valueOrDefault(env.apply("APUS_RUNNER_IMAGE"), DEFAULT_RUNNER_IMAGE)); + valueOrDefault(env.apply("APUS_RUNNER_IMAGE"), DEFAULT_RUNNER_IMAGE), + valueOrDefault(env.apply("APUS_INGEST_IMAGE"), DEFAULT_INGEST_IMAGE), + valueOrDefault(env.apply("APUS_BUNDLE_BUCKET"), DEFAULT_BUNDLE_BUCKET), + valueOrDefault(env.apply("APUS_BUNDLE_S3_ENDPOINT"), DEFAULT_BUNDLE_S3_ENDPOINT), + valueOrDefault(env.apply("APUS_BUNDLE_S3_REGION"), DEFAULT_BUNDLE_S3_REGION), + valueOrDefault(env.apply("APUS_BUNDLE_CREDENTIALS_SECRET"), DEFAULT_BUNDLE_CREDENTIALS_SECRET)); } private static String valueOrDefault(String value, String defaultValue) { diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/api/BundleRef.java b/operator/src/main/java/net/onelitefeather/apus/operator/api/BundleRef.java new file mode 100644 index 0000000..1cea125 --- /dev/null +++ b/operator/src/main/java/net/onelitefeather/apus/operator/api/BundleRef.java @@ -0,0 +1,60 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.api; + +import java.util.ArrayList; +import java.util.List; + +/** + * A reference to a specific, already-ingested world bundle: where it lives, which version it + * is, and which dimensions it contains. + * + *

Shared between {@link WorldSourceStatus} (the latest bundle a source has produced) and + * {@link WorldIngestStatus} (the bundle a single ingest run produced), so both statuses describe + * "a bundle" the exact same way. + */ +public class BundleRef { + + private String path; + private String version; + private List dimensions = new ArrayList<>(); + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public List getDimensions() { + return dimensions; + } + + public void setDimensions(List dimensions) { + this.dimensions = dimensions; + } +} diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/api/Labels.java b/operator/src/main/java/net/onelitefeather/apus/operator/api/Labels.java index ec5c51f..287d380 100644 --- a/operator/src/main/java/net/onelitefeather/apus/operator/api/Labels.java +++ b/operator/src/main/java/net/onelitefeather/apus/operator/api/Labels.java @@ -70,6 +70,21 @@ public final class Labels { */ public static final String MAP_UID = "apus.onelitefeather.net/map-uid"; + /** + * The {@code WorldSource} a per-source resource (e.g. a {@code WorldIngest} created by + * {@code WorldSourceReconciler}) belongs to, by name. Mirrors {@link #MAP}: not unique on + * its own once a source can be deleted and recreated with the same name -- see {@link + * #SOURCE_UID}. + */ + public static final String SOURCE = "apus.onelitefeather.net/world-source"; + + /** + * The UID of the owning {@code WorldSource} resource. Mirrors {@link #MAP_UID}: a source + * name can be reused after deletion, but its UID never is, so ownership checks must + * compare this label, not just {@link #SOURCE}. + */ + public static final String SOURCE_UID = "apus.onelitefeather.net/world-source-uid"; + private Labels() {} /** diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/api/WorldIngest.java b/operator/src/main/java/net/onelitefeather/apus/operator/api/WorldIngest.java new file mode 100644 index 0000000..829047d --- /dev/null +++ b/operator/src/main/java/net/onelitefeather/apus/operator/api/WorldIngest.java @@ -0,0 +1,49 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.api; + +import io.fabric8.kubernetes.api.model.Namespaced; +import io.fabric8.kubernetes.client.CustomResource; +import io.fabric8.kubernetes.model.annotation.Group; +import io.fabric8.kubernetes.model.annotation.Kind; +import io.fabric8.kubernetes.model.annotation.Plural; +import io.fabric8.kubernetes.model.annotation.ShortNames; +import io.fabric8.kubernetes.model.annotation.Version; + +/** + * A single ingest run: extracts one world at one version out of a {@link WorldSource} and + * transforms/loads it into the common bundle format {@link BlueMapMap} renders from. Namespaced: + * an ingest run belongs to exactly one tenant's namespace, exactly like {@link WorldSource}. + */ +@Group("bluemap.onelitefeather.net") +@Version("v1alpha1") +@Kind("WorldIngest") +@Plural("worldingests") +@ShortNames("bmingest") +public class WorldIngest extends CustomResource implements Namespaced { + + @Override + protected WorldIngestSpec initSpec() { + return new WorldIngestSpec(); + } + + @Override + protected WorldIngestStatus initStatus() { + return new WorldIngestStatus(); + } +} diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/api/WorldIngestSpec.java b/operator/src/main/java/net/onelitefeather/apus/operator/api/WorldIngestSpec.java new file mode 100644 index 0000000..5c0c650 --- /dev/null +++ b/operator/src/main/java/net/onelitefeather/apus/operator/api/WorldIngestSpec.java @@ -0,0 +1,55 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.api; + +/** + * Desired state of a {@link WorldIngest}. Plain data, no Kubernetes access. + * + *

{@code sourceRef} is initialised in its field declaration so a reconciler never has to + * null-check its way down to a leaf field. + */ +public class WorldIngestSpec { + + private Ref sourceRef = new Ref(); + private String sourceVersion; + private String worldName; + + public Ref getSourceRef() { + return sourceRef; + } + + public void setSourceRef(Ref sourceRef) { + this.sourceRef = sourceRef; + } + + public String getSourceVersion() { + return sourceVersion; + } + + public void setSourceVersion(String sourceVersion) { + this.sourceVersion = sourceVersion; + } + + public String getWorldName() { + return worldName; + } + + public void setWorldName(String worldName) { + this.worldName = worldName; + } +} diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/api/WorldIngestStatus.java b/operator/src/main/java/net/onelitefeather/apus/operator/api/WorldIngestStatus.java new file mode 100644 index 0000000..55003c3 --- /dev/null +++ b/operator/src/main/java/net/onelitefeather/apus/operator/api/WorldIngestStatus.java @@ -0,0 +1,126 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.api; + +import io.fabric8.kubernetes.api.model.Condition; +import java.util.ArrayList; +import java.util.List; + +/** + * Observed state of a {@link WorldIngest}. Every group is initialised in its field declaration + * so a reconciler never has to null-check its way down to a leaf field. + */ +public class WorldIngestStatus { + + /** Pending|Extracting|Transforming|Loading|Succeeded|Failed */ + private String phase; + + private Progress progress = new Progress(); + private BundleRef bundle = new BundleRef(); + private String jobName; + private String startTime; + private String completionTime; + private List conditions = new ArrayList<>(); + + public String getPhase() { + return phase; + } + + public void setPhase(String phase) { + this.phase = phase; + } + + public Progress getProgress() { + return progress; + } + + public void setProgress(Progress progress) { + this.progress = progress; + } + + public BundleRef getBundle() { + return bundle; + } + + public void setBundle(BundleRef bundle) { + this.bundle = bundle; + } + + public String getJobName() { + return jobName; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public String getStartTime() { + return startTime; + } + + public void setStartTime(String startTime) { + this.startTime = startTime; + } + + public String getCompletionTime() { + return completionTime; + } + + public void setCompletionTime(String completionTime) { + this.completionTime = completionTime; + } + + public List getConditions() { + return conditions; + } + + public void setConditions(List conditions) { + this.conditions = conditions; + } + + /** How far the current ingest run has gotten. */ + public static class Progress { + private double percent; + private long bytesDone; + private long bytesTotal; + + public double getPercent() { + return percent; + } + + public void setPercent(double percent) { + this.percent = percent; + } + + public long getBytesDone() { + return bytesDone; + } + + public void setBytesDone(long bytesDone) { + this.bytesDone = bytesDone; + } + + public long getBytesTotal() { + return bytesTotal; + } + + public void setBytesTotal(long bytesTotal) { + this.bytesTotal = bytesTotal; + } + } +} diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/api/WorldSource.java b/operator/src/main/java/net/onelitefeather/apus/operator/api/WorldSource.java new file mode 100644 index 0000000..7b6fd38 --- /dev/null +++ b/operator/src/main/java/net/onelitefeather/apus/operator/api/WorldSource.java @@ -0,0 +1,99 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.api; + +import io.fabric8.kubernetes.api.model.Namespaced; +import io.fabric8.kubernetes.client.CustomResource; +import io.fabric8.kubernetes.model.annotation.Group; +import io.fabric8.kubernetes.model.annotation.Kind; +import io.fabric8.kubernetes.model.annotation.Plural; +import io.fabric8.kubernetes.model.annotation.ShortNames; +import io.fabric8.kubernetes.model.annotation.Version; + +/** + * A source of Minecraft world data that Apus can ingest -- an S3 bucket, a Pterodactyl panel, a + * manual upload, or a push target. Namespaced: a source belongs to exactly one tenant's + * namespace, exactly like {@link BlueMapMap}. + */ +@Group("bluemap.onelitefeather.net") +@Version("v1alpha1") +@Kind("WorldSource") +@Plural("worldsources") +@ShortNames("bmsource") +public class WorldSource extends CustomResource implements Namespaced { + + @Override + protected WorldSourceSpec initSpec() { + return new WorldSourceSpec(); + } + + @Override + protected WorldSourceStatus initStatus() { + return new WorldSourceStatus(); + } + + /** One world this source exposes for ingest, and how its on-disk layout should be detected. */ + public static class WorldSelector { + + private String name; + + /** "auto" makes the ingest job detect the layout (vanilla/Paper/multiverse/...) itself. */ + private String layout = "auto"; + + /** + * The Minecraft version this world runs under, e.g. {@code "1.21.10"} -- recorded + * verbatim into {@code manifest.minecraftVersion} for every bundle ingested from this + * selector. + * + *

Why this lives here instead of being read from {@code level.dat}. {@code + * level.dat} is NBT, not JSON/plain text, and this project intentionally does not carry + * an NBT parsing dependency (see {@code ingest/README.md}'s connector-only dependency + * policy). Since {@code level.dat} is bundled starting from this same fix (see the + * ingest layer's D1 fix), a future change could still read it back out once an NBT + * reader exists -- but a required manifest field cannot stay permanently unset waiting + * for that; a user-supplied value the tenant already knows (the version they run) is the + * simplest correct answer available today. {@code null}/unset means the field is left + * absent on the bundle, exactly as it always has been -- this is purely additive. + */ + private String minecraftVersion; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getLayout() { + return layout; + } + + public void setLayout(String layout) { + this.layout = layout; + } + + public String getMinecraftVersion() { + return minecraftVersion; + } + + public void setMinecraftVersion(String minecraftVersion) { + this.minecraftVersion = minecraftVersion; + } + } +} diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/api/WorldSourceSpec.java b/operator/src/main/java/net/onelitefeather/apus/operator/api/WorldSourceSpec.java new file mode 100644 index 0000000..cdaee89 --- /dev/null +++ b/operator/src/main/java/net/onelitefeather/apus/operator/api/WorldSourceSpec.java @@ -0,0 +1,185 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.api; + +import java.util.ArrayList; +import java.util.List; + +/** + * Desired state of a {@link WorldSource}. Plain data, no Kubernetes access. + * + *

Every group is initialised in its field declaration so a reconciler (or a test) never has + * to null-check its way down to a leaf field. + */ +public class WorldSourceSpec { + + /** "s3" | "pterodactyl" | "upload" | "push" */ + private String type; + + private S3Source s3 = new S3Source(); + private Pterodactyl pterodactyl = new Pterodactyl(); + + /** Cron expression driving polling for pull-based source types; null means manual only. */ + private String poll; + + private List worlds = new ArrayList<>(); + private Retention retention = new Retention(); + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public S3Source getS3() { + return s3; + } + + public void setS3(S3Source s3) { + this.s3 = s3; + } + + public Pterodactyl getPterodactyl() { + return pterodactyl; + } + + public void setPterodactyl(Pterodactyl pterodactyl) { + this.pterodactyl = pterodactyl; + } + + public String getPoll() { + return poll; + } + + public void setPoll(String poll) { + this.poll = poll; + } + + public List getWorlds() { + return worlds; + } + + public void setWorlds(List worlds) { + this.worlds = worlds; + } + + public Retention getRetention() { + return retention; + } + + public void setRetention(Retention retention) { + this.retention = retention; + } + + /** Connection details for an S3-compatible bucket backing this source. */ + public static class S3Source { + private String endpoint; + private String bucket; + private String prefix; + private Ref credentialsSecretRef = new Ref(); + + public String getEndpoint() { + return endpoint; + } + + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } + + public String getBucket() { + return bucket; + } + + public void setBucket(String bucket) { + this.bucket = bucket; + } + + public String getPrefix() { + return prefix; + } + + public void setPrefix(String prefix) { + this.prefix = prefix; + } + + public Ref getCredentialsSecretRef() { + return credentialsSecretRef; + } + + public void setCredentialsSecretRef(Ref credentialsSecretRef) { + this.credentialsSecretRef = credentialsSecretRef; + } + } + + /** Connection details for a Pterodactyl panel backing this source. */ + public static class Pterodactyl { + private String panelUrl; + private String serverId; + private Ref credentialsSecretRef = new Ref(); + + /** "latest" makes the ingest job pick the most recent backup/world archive itself. */ + private String select = "latest"; + + public String getPanelUrl() { + return panelUrl; + } + + public void setPanelUrl(String panelUrl) { + this.panelUrl = panelUrl; + } + + public String getServerId() { + return serverId; + } + + public void setServerId(String serverId) { + this.serverId = serverId; + } + + public Ref getCredentialsSecretRef() { + return credentialsSecretRef; + } + + public void setCredentialsSecretRef(Ref credentialsSecretRef) { + this.credentialsSecretRef = credentialsSecretRef; + } + + public String getSelect() { + return select; + } + + public void setSelect(String select) { + this.select = select; + } + } + + /** How many past bundle versions this source retains before older ones are pruned. */ + public static class Retention { + private int keepVersions = 5; + + public int getKeepVersions() { + return keepVersions; + } + + public void setKeepVersions(int keepVersions) { + this.keepVersions = keepVersions; + } + } +} diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/api/WorldSourceStatus.java b/operator/src/main/java/net/onelitefeather/apus/operator/api/WorldSourceStatus.java new file mode 100644 index 0000000..ce13698 --- /dev/null +++ b/operator/src/main/java/net/onelitefeather/apus/operator/api/WorldSourceStatus.java @@ -0,0 +1,106 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.api; + +import io.fabric8.kubernetes.api.model.Condition; +import java.util.ArrayList; +import java.util.List; + +/** + * Observed state of a {@link WorldSource}. Every group is initialised in its field declaration + * so a reconciler never has to null-check its way down to a leaf field. + */ +public class WorldSourceStatus { + + private String lastSeenVersion; + private BundleRef latestBundle = new BundleRef(); + private String lastPollTime; + + /** + * The optimistic lock {@code WorldIngestReconciler} claims before submitting an ingest Job, + * so two {@code WorldIngest} runs for this source never write to the bundle bucket at once. + * Exactly mirrors {@link BlueMapMapStatus#getLatestRender()} -- see {@code + * BlueMapRenderReconciler}'s class Javadoc for why an optimistic {@code updateStatus()} on + * the referenced resource (not the run itself) is the only race-free way to enforce this. + */ + private ActiveIngest activeIngest = new ActiveIngest(); + + private List conditions = new ArrayList<>(); + + public String getLastSeenVersion() { + return lastSeenVersion; + } + + public void setLastSeenVersion(String lastSeenVersion) { + this.lastSeenVersion = lastSeenVersion; + } + + public BundleRef getLatestBundle() { + return latestBundle; + } + + public void setLatestBundle(BundleRef latestBundle) { + this.latestBundle = latestBundle; + } + + public String getLastPollTime() { + return lastPollTime; + } + + public void setLastPollTime(String lastPollTime) { + this.lastPollTime = lastPollTime; + } + + public ActiveIngest getActiveIngest() { + return activeIngest; + } + + public void setActiveIngest(ActiveIngest activeIngest) { + this.activeIngest = activeIngest; + } + + public List getConditions() { + return conditions; + } + + public void setConditions(List conditions) { + this.conditions = conditions; + } + + /** The most recent {@link WorldIngest} triggered for this source. */ + public static class ActiveIngest { + private String name; + private String phase; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getPhase() { + return phase; + } + + public void setPhase(String phase) { + this.phase = phase; + } + } +} diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/ingest/AwsBundleStore.java b/operator/src/main/java/net/onelitefeather/apus/operator/ingest/AwsBundleStore.java new file mode 100644 index 0000000..e580a95 --- /dev/null +++ b/operator/src/main/java/net/onelitefeather/apus/operator/ingest/AwsBundleStore.java @@ -0,0 +1,111 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.ingest; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import net.onelitefeather.apus.ingest.BundlePath; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CommonPrefix; +import software.amazon.awssdk.services.s3.model.Delete; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.S3Object; + +/** + * Real {@link BundleStore} backed by an AWS SDK v2 {@link S3Client} -- the same client family + * {@code net.onelitefeather.apus.ingest.S3Client} wraps for writing bundles, used here in the + * operator to enforce {@code WorldSource.spec.retention}. + * + *

Deliberately has no dedicated unit test: it is a thin, directly-inspectable wrapper around + * two SDK calls, exactly the same shape (and the same reasoning) as {@code + * net.onelitefeather.apus.ingest.S3Client#wrapping} -- the interface {@link BundleStore} is + * fully exercised by {@link WorldIngestReconcilerTest} through an in-memory fake instead. + * + *

"Version" = the common prefix directly under {@link BundlePath#prefix}. {@link + * net.onelitefeather.apus.ingest.BundleWriter#write} always writes a bundle under exactly that + * shape ({@code ////...}), so listing common prefixes one + * level deep is enough to enumerate every version without inspecting individual object keys. + */ +public final class AwsBundleStore implements BundleStore { + + /** S3's own limit on how many keys a single {@code DeleteObjects} call may name. */ + private static final int DELETE_BATCH_SIZE = 1000; + + private final S3Client client; + + public AwsBundleStore(S3Client client) { + this.client = client; + } + + @Override + public List listVersions(String tenant, String sourceName, String worldId, String bundleBucket) { + String prefix = BundlePath.prefix(tenant, sourceName, worldId); + ListObjectsV2Request request = ListObjectsV2Request.builder() + .bucket(bundleBucket) + .prefix(prefix) + .delimiter("/") + .build(); + + List versions = new ArrayList<>(); + for (CommonPrefix commonPrefix : client.listObjectsV2Paginator(request).commonPrefixes()) { + String versionPrefix = commonPrefix.prefix(); // "////" + String version = versionPrefix.substring(prefix.length(), versionPrefix.length() - 1); + versions.add(new BundleVersion(version, lastModifiedOf(bundleBucket, versionPrefix))); + } + return versions; + } + + /** + * The version's own "last written" timestamp is taken from its {@code manifest.json} -- + * written last by {@link net.onelitefeather.apus.ingest.BundleWriter#write}, so its + * timestamp is also the version's actual completion time, not merely the first region file + * uploaded. + */ + private Instant lastModifiedOf(String bundleBucket, String versionPrefix) { + ListObjectsV2Request request = ListObjectsV2Request.builder() + .bucket(bundleBucket) + .prefix(versionPrefix + "manifest.json") + .build(); + return client.listObjectsV2(request).contents().stream() + .findFirst() + .map(S3Object::lastModified) + .orElse(Instant.EPOCH); + } + + @Override + public void deleteVersion(String tenant, String sourceName, String worldId, String version, String bundleBucket) { + String prefix = BundlePath.of(tenant, sourceName, worldId, version) + "/"; + List keys = new ArrayList<>(); + ListObjectsV2Request listRequest = + ListObjectsV2Request.builder().bucket(bundleBucket).prefix(prefix).build(); + for (S3Object object : client.listObjectsV2Paginator(listRequest).contents()) { + keys.add(ObjectIdentifier.builder().key(object.key()).build()); + } + + for (int start = 0; start < keys.size(); start += DELETE_BATCH_SIZE) { + List batch = keys.subList(start, Math.min(start + DELETE_BATCH_SIZE, keys.size())); + client.deleteObjects(DeleteObjectsRequest.builder() + .bucket(bundleBucket) + .delete(Delete.builder().objects(batch).build()) + .build()); + } + } +} diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/ingest/BundleStore.java b/operator/src/main/java/net/onelitefeather/apus/operator/ingest/BundleStore.java new file mode 100644 index 0000000..1f999fe --- /dev/null +++ b/operator/src/main/java/net/onelitefeather/apus/operator/ingest/BundleStore.java @@ -0,0 +1,48 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.ingest; + +import java.time.Instant; +import java.util.List; + +/** + * The two bundle-bucket operations {@link WorldIngestReconciler}'s retention enforcement needs, + * kept deliberately narrow -- mirrors {@code net.onelitefeather.apus.ingest.S3Client}'s "one + * interface, real implementation plus an easily fakeable one for tests" shape -- so a test can + * substitute an in-memory fake instead of talking to real S3-compatible storage. + */ +public interface BundleStore { + + /** One bundle version found under a world's prefix in the bucket. */ + record BundleVersion(String version, Instant lastModified) {} + + /** + * Lists every bundle version currently written for {@code tenant}/{@code sourceName}/{@code + * worldId}, in no particular order -- callers sort as needed. + * + *

{@code sourceName} scopes the listing to one {@code WorldSource}'s own bundles -- see + * {@code net.onelitefeather.apus.ingest.BundlePath} for why {@code worldId} alone (the + * Minecraft world's own directory name, commonly the vanilla default {@code "world"}) is not + * enough to identify a unique bundle lineage: two different sources in the same namespace can + * both use it. + */ + List listVersions(String tenant, String sourceName, String worldId, String bundleBucket); + + /** Deletes every object under one bundle version's prefix. */ + void deleteVersion(String tenant, String sourceName, String worldId, String version, String bundleBucket); +} diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/ingest/CronSchedule.java b/operator/src/main/java/net/onelitefeather/apus/operator/ingest/CronSchedule.java new file mode 100644 index 0000000..a8d7fd4 --- /dev/null +++ b/operator/src/main/java/net/onelitefeather/apus/operator/ingest/CronSchedule.java @@ -0,0 +1,100 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.ingest; + +import com.cronutils.model.Cron; +import com.cronutils.model.CronType; +import com.cronutils.model.definition.CronDefinitionBuilder; +import com.cronutils.model.time.ExecutionTime; +import com.cronutils.parser.CronParser; +import java.time.Duration; +import java.time.ZonedDateTime; +import java.util.Optional; + +/** + * Evaluates a {@code WorldSourceSpec.poll} Cron expression: whether a poll is due given when the + * source was last polled, and how long until the next one is. + * + *

Backed by cron-utils rather than a + * hand-rolled parser -- see {@code settings.gradle.kts} for why that library and version were + * chosen. Uses the standard five-field Unix cron syntax ({@code minute hour day-of-month month + * day-of-week}, no seconds field), matching Kubernetes' own {@code CronJob.spec.schedule} so an + * operator-facing {@code WorldSource.spec.poll} value reads exactly like a value the same person + * would already know from writing a {@code CronJob}. + */ +public final class CronSchedule { + + private static final CronParser PARSER = + new CronParser(CronDefinitionBuilder.instanceDefinitionFor(CronType.UNIX)); + + /** + * Used when no poll has happened yet and there is therefore no {@code timeToNextExecution} + * anchor point other than "now" -- a poll is due immediately in that case (see {@link + * #isDue}), so this only matters for {@link #timeToNext}, where it is a conservative, + * short fallback rather than leaving the reconciler unscheduled. + */ + private static final Duration FALLBACK_RECHECK = Duration.ofMinutes(1); + + private final ExecutionTime executionTime; + + private CronSchedule(ExecutionTime executionTime) { + this.executionTime = executionTime; + } + + /** + * Parses {@code expression} as a five-field Unix cron expression. + * + * @throws InvalidCronExpressionException if the expression is syntactically invalid + */ + public static CronSchedule parse(String expression) { + try { + Cron cron = PARSER.parse(expression); + cron.validate(); + return new CronSchedule(ExecutionTime.forCron(cron)); + } catch (IllegalArgumentException e) { + throw new InvalidCronExpressionException( + "invalid poll cron expression '" + expression + "': " + e.getMessage(), e); + } + } + + /** + * Whether a poll is due: {@code true} if this source has never been polled ({@code lastPoll} + * is {@code null} -- nothing to wait for), or if this schedule's next execution after {@code + * lastPoll} falls at or before {@code now}. + */ + public boolean isDue(ZonedDateTime lastPoll, ZonedDateTime now) { + if (lastPoll == null) { + return true; + } + Optional next = executionTime.nextExecution(lastPoll); + return next.isPresent() && !next.get().isAfter(now); + } + + /** How long from {@code now} until this schedule's next execution. */ + public Duration timeToNext(ZonedDateTime now) { + return executionTime.timeToNextExecution(now).orElse(FALLBACK_RECHECK); + } + + /** Thrown by {@link #parse} when the given string is not a valid Cron expression. */ + public static final class InvalidCronExpressionException extends RuntimeException { + + InvalidCronExpressionException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/ingest/IngestJobBuilder.java b/operator/src/main/java/net/onelitefeather/apus/operator/ingest/IngestJobBuilder.java new file mode 100644 index 0000000..e9120e8 --- /dev/null +++ b/operator/src/main/java/net/onelitefeather/apus/operator/ingest/IngestJobBuilder.java @@ -0,0 +1,383 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.ingest; + +import io.fabric8.kubernetes.api.model.Container; +import io.fabric8.kubernetes.api.model.ContainerBuilder; +import io.fabric8.kubernetes.api.model.EnvVar; +import io.fabric8.kubernetes.api.model.EnvVarBuilder; +import io.fabric8.kubernetes.api.model.OwnerReference; +import io.fabric8.kubernetes.api.model.OwnerReferenceBuilder; +import io.fabric8.kubernetes.api.model.Quantity; +import io.fabric8.kubernetes.api.model.ResourceRequirements; +import io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder; +import io.fabric8.kubernetes.api.model.batch.v1.Job; +import io.fabric8.kubernetes.api.model.batch.v1.JobBuilder; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import net.onelitefeather.apus.operator.OperatorConfig; +import net.onelitefeather.apus.operator.api.Labels; +import net.onelitefeather.apus.operator.api.WorldIngest; +import net.onelitefeather.apus.operator.api.WorldSource; + +/** + * Turns a {@link WorldIngest} plus the {@link WorldSource} it targets into the Kubernetes {@link + * Job} that actually performs the ingest, by driving the {@code apus/ingest} image (phase 2b, + * task 5) through its environment-variable contract ({@code ingest/README.md}). + * + *

Pure function: no Kubernetes client, no side effects, exactly like {@code RenderJobBuilder} + * -- the caller ({@link WorldIngestReconciler}) is responsible for actually submitting the + * returned {@link Job} and for having already claimed the concurrency lock this builder assumes + * is held. + * + *

Bundle destination is operator-wide, not per-source. Neither {@link WorldSource} nor + * {@link WorldIngest} carries a bundle-bucket field -- only {@code WorldSourceSpec.s3}/{@code + * .pterodactyl}, which describe the raw data *source*, not where the resulting bundle is written. + * The destination therefore comes from {@link OperatorConfig#bundleBucket()} and friends, the + * same site-wide-setting pattern {@link OperatorConfig#runnerImage()} already established. See + * that record's Javadoc for the full reasoning. + * + *

Bundle version identifier. {@code APUS_BUNDLE_VERSION} must be distinct from {@code + * APUS_SOURCE_VERSION} ({@code ingest/README.md}'s own run example uses {@code v1} for the + * former and a source-specific, potentially messy id for the latter) -- a bundle version needs a + * clean, unique-per-run identifier a render can reference and retention can enumerate. This + * builder uses the {@link WorldIngest}'s own resource name: {@link WorldSourceReconciler} already + * mints a fresh, unique name per discovered source version, so reusing it costs nothing extra and + * keeps the ingest run and the bundle it produces traceable to each other by the same string. + * + *

Bounded ephemeral storage. The ingest container mounts no volume for its work + * directory or the archive it extracts -- both land on the container's writable layer, backed by + * the node's own disk. {@code Archives}' own configurable total-bytes/entry-count limits (see + * {@code ingest/README.md}) stop a hostile "archive bomb" from writing unbounded data, but without + * a Kubernetes-level {@code ephemeral-storage} resource limit too, even a *legitimate* large world + * could still starve the node's disk for every other pod scheduled on it. This builder therefore + * always sets both a request and a limit for it, as defense in depth alongside the application-level + * check, not instead of it. + */ +public final class IngestJobBuilder { + + /** API group + version the owning {@link WorldIngest} is served under. */ + private static final String OWNER_API_VERSION = "bluemap.onelitefeather.net/v1alpha1"; + + private static final String OWNER_KIND = "WorldIngest"; + + /** Mirrors {@code RenderJobBuilder.BACKOFF_LIMIT}'s reasoning: fail fast, don't retry forever. */ + private static final int BACKOFF_LIMIT = 2; + + private static final String CONTAINER_NAME = "ingest"; + + /** Mirrors {@code RenderJobBuilder.TERMINATION_MESSAGE_POLICY} -- see its Javadoc. */ + private static final String TERMINATION_MESSAGE_POLICY = "FallbackToLogsOnError"; + + private static final String AUTO_LAYOUT = "auto"; + private static final String TYPE_S3 = "s3"; + private static final String TYPE_PTERODACTYL = "pterodactyl"; + + /** + * Ephemeral-storage request/limit for the ingest container -- see the class Javadoc's + * "Bounded ephemeral storage" section for why this exists at all. + */ + private static final String EPHEMERAL_STORAGE_REQUEST = "2Gi"; + + private static final String EPHEMERAL_STORAGE_LIMIT = "10Gi"; + + /** Secret data key expected on the bundle destination credentials secret. */ + private static final String BUNDLE_ACCESS_KEY = "AWS_ACCESS_KEY_ID"; + + private static final String BUNDLE_SECRET_KEY = "AWS_SECRET_ACCESS_KEY"; + + /** + * Secret data key expected on {@code WorldSourceSpec.S3Source.credentialsSecretRef} when + * set; matches the destination convention above so one Secret shape works for both roles. + */ + private static final String SOURCE_S3_ACCESS_KEY = "AWS_ACCESS_KEY_ID"; + + private static final String SOURCE_S3_SECRET_KEY = "AWS_SECRET_ACCESS_KEY"; + + /** Secret data key expected on {@code WorldSourceSpec.Pterodactyl.credentialsSecretRef}. */ + private static final String PTERODACTYL_API_KEY = "API_KEY"; + + private IngestJobBuilder() {} + + /** + * Builds the ingest {@link Job} for one {@link WorldIngest} run. + * + * @param ingest the ingest run to execute; supplies the world name, source version and owns + * the returned job via an owner reference + * @param source the {@link WorldSource} being pulled from; supplies the source type and + * connection details + * @param config operator-wide settings: the ingest image and the bundle destination + * @return the {@link Job} manifest, not yet submitted to the API server + */ + public static Job build(WorldIngest ingest, WorldSource source, OperatorConfig config) { + String namespace = ingest.getMetadata().getNamespace(); + Map labels = labels(ingest, source); + + Container container = new ContainerBuilder() + .withName(CONTAINER_NAME) + .withImage(config.ingestImage()) + .withEnv(env(ingest, source, config)) + .withResources(resources()) + .withTerminationMessagePolicy(TERMINATION_MESSAGE_POLICY) + .build(); + + return new JobBuilder() + .withNewMetadata() + .withName(ingest.getMetadata().getName()) + .withNamespace(namespace) + .withLabels(labels) + .withOwnerReferences(ownerReference(ingest)) + .endMetadata() + .withNewSpec() + .withBackoffLimit(BACKOFF_LIMIT) + .withNewTemplate() + .withNewMetadata() + .withLabels(labels) + .endMetadata() + .withNewSpec() + .withRestartPolicy("Never") + .withContainers(container) + .endSpec() + .endTemplate() + .endSpec() + .build(); + } + + /** + * The bundle's version identifier this job will write under -- see the class Javadoc's + * "Bundle version identifier" section. Exposed so {@link WorldIngestReconciler} can compute + * {@code WorldSource.status.latestBundle}/{@code WorldIngest.status.bundle} without having + * to duplicate or parse it back out of the job. + */ + public static String bundleVersion(WorldIngest ingest) { + return ingest.getMetadata().getName(); + } + + private static Map labels(WorldIngest ingest, WorldSource source) { + Map labels = Labels.standard("world-ingest", ingest.getMetadata().getName()); + labels.put(Labels.SOURCE, source.getMetadata().getName()); + if (source.getMetadata().getUid() != null) { + labels.put(Labels.SOURCE_UID, source.getMetadata().getUid()); + } + return labels; + } + + /** + * The ingest container's {@code ephemeral-storage} request/limit -- see the class Javadoc's + * "Bounded ephemeral storage" section. + */ + private static ResourceRequirements resources() { + Map quantities = new LinkedHashMap<>(); + quantities.put("ephemeral-storage", new Quantity(EPHEMERAL_STORAGE_REQUEST)); + Map limits = new LinkedHashMap<>(); + limits.put("ephemeral-storage", new Quantity(EPHEMERAL_STORAGE_LIMIT)); + return new ResourceRequirementsBuilder() + .withRequests(quantities) + .withLimits(limits) + .build(); + } + + private static OwnerReference ownerReference(WorldIngest ingest) { + return new OwnerReferenceBuilder() + .withApiVersion(OWNER_API_VERSION) + .withKind(OWNER_KIND) + .withName(ingest.getMetadata().getName()) + .withUid(ingest.getMetadata().getUid()) + .withController(true) + .withBlockOwnerDeletion(true) + .build(); + } + + /** + * Builds the environment for the {@code ingest} container to satisfy the phase 2b ingest + * image's contract exactly ({@code ingest/README.md}). Every mandatory variable is always + * set; optional ones are only added when the data model actually carries a value, so the + * image's own defaults apply otherwise. + */ + private static List env(WorldIngest ingest, WorldSource source, OperatorConfig config) { + String namespace = ingest.getMetadata().getNamespace(); + String worldName = ingest.getSpec().getWorldName(); + String bundleVersion = bundleVersion(ingest); + + List env = new ArrayList<>(); + + // Mandatory -- IngestConfig.fromEnv exits non-zero at startup if any of these is missing. + env.add(literal("APUS_SOURCE_TYPE", source.getSpec().getType())); + env.add(literal("APUS_WORLD_NAME", worldName)); + env.add(literal("APUS_SOURCE_VERSION", ingest.getSpec().getSourceVersion())); + env.add(literal("APUS_BUNDLE_BUCKET", config.bundleBucket())); + env.add(literal("APUS_BUNDLE_TENANT", tenantNameForNamespace(namespace))); + // Scopes the bundle path by the owning source's name, not just worldId -- see + // net.onelitefeather.apus.ingest.BundlePath's Javadoc for why worldId alone (the + // Minecraft world's own directory name, commonly the vanilla default "world") is not + // enough to keep two different sources' bundles from colliding on the same prefix. + env.add(literal("APUS_BUNDLE_SOURCE_NAME", source.getMetadata().getName())); + env.add(literal("APUS_BUNDLE_WORLD_ID", worldName)); + env.add(literal("APUS_BUNDLE_VERSION", bundleVersion)); + env.add(literal("APUS_S3_ENDPOINT", config.bundleS3Endpoint())); + env.add(fromSecret("APUS_S3" + "_ACCESS_KEY", config.bundleCredentialsSecretName(), BUNDLE_ACCESS_KEY)); + env.add(fromSecret("APUS_S3" + "_SECRET_KEY", config.bundleCredentialsSecretName(), BUNDLE_SECRET_KEY)); + + // Optional -- only set when the CR/config actually carries a non-default value. + if (config.bundleS3Region() != null && !config.bundleS3Region().isBlank()) { + env.add(literal("APUS_S3_REGION", config.bundleS3Region())); + } + env.add(literal("APUS_LAYOUT", layoutFor(source, worldName))); + String minecraftVersion = minecraftVersionFor(source, worldName); + if (minecraftVersion != null && !minecraftVersion.isBlank()) { + env.add(literal("APUS_MC_VERSION", minecraftVersion)); + } + + switch (source.getSpec().getType()) { + case TYPE_S3 -> env.addAll(s3SourceEnv(source)); + case TYPE_PTERODACTYL -> env.addAll(pterodactylSourceEnv(source, worldName)); + default -> { + // WorldSourceReconciler never creates a WorldIngest for an unsupported source + // type (upload/push have no connector yet -- phase 6), so reaching this means + // the two disagree about which types are pollable/ingestible. + } + } + + return env; + } + + private static List s3SourceEnv(WorldSource source) { + List env = new ArrayList<>(); + WorldSourceSpecAccess s3 = WorldSourceSpecAccess.s3(source); + env.add(literal("APUS_SOURCE_S3_BUCKET", s3.bucket())); + addIfPresent(env, "APUS_SOURCE_S3_ENDPOINT", s3.endpoint()); + addIfPresent(env, "APUS_SOURCE_S3_PREFIX", s3.prefix()); + if (s3.credentialsSecretName() != null && !s3.credentialsSecretName().isBlank()) { + env.add(fromSecret("APUS_SOURCE_S3_ACCESS_KEY", s3.credentialsSecretName(), SOURCE_S3_ACCESS_KEY)); + env.add(fromSecret("APUS_SOURCE_S3_SECRET_KEY", s3.credentialsSecretName(), SOURCE_S3_SECRET_KEY)); + } + return env; + } + + /** + * {@code APUS_PTERODACTYL_WORLD_PATHS} names the top-level backup archive paths that make up + * one logical world -- needed because {@code PterodactylConnector.fetch} streams a whole-server + * {@code tar.gz} exactly once and writes only matching entries (see its Javadoc). Neither + * {@code WorldSourceSpec} nor {@code WorldIngestSpec} carries this breakdown explicitly, so + * this derives it from the project's own established Bukkit split-world convention (the same + * one {@code LayoutDetector}/the phase 2b plan document: {@code }, {@code + * _nether}, {@code _the_end}). A vanilla single-directory world is simply the + * first of the three paths with the other two never matching anything in the archive, which + * is harmless. A future {@code WorldSelector} field could override this if a server uses a + * non-standard split -- not needed by any fixture or spec this phase defines. + */ + private static List pterodactylSourceEnv(WorldSource source, String worldName) { + List env = new ArrayList<>(); + WorldSourceSpecAccess pterodactyl = WorldSourceSpecAccess.pterodactyl(source); + env.add(literal("APUS_PTERODACTYL_PANEL_URL", pterodactyl.panelUrl())); + env.add(literal("APUS_PTERODACTYL_SERVER_ID", pterodactyl.serverId())); + env.add(fromSecret("APUS_PTERODACTYL_API_KEY", pterodactyl.credentialsSecretName(), PTERODACTYL_API_KEY)); + env.add(literal( + "APUS_PTERODACTYL_WORLD_PATHS", + worldName + "," + worldName + "_nether" + "," + worldName + "_the_end")); + return env; + } + + private static void addIfPresent(List env, String name, String value) { + if (value != null && !value.isBlank()) { + env.add(literal(name, value)); + } + } + + private static String layoutFor(WorldSource source, String worldName) { + for (WorldSource.WorldSelector selector : source.getSpec().getWorlds()) { + if (worldName != null && worldName.equals(selector.getName())) { + return selector.getLayout(); + } + } + return AUTO_LAYOUT; + } + + /** + * The Minecraft version configured on the matching {@link WorldSource.WorldSelector}, or + * {@code null} if no selector matches or none was configured -- see {@link + * WorldSource.WorldSelector#getMinecraftVersion()} for why this is a user-supplied field + * rather than read from {@code level.dat}. + */ + private static String minecraftVersionFor(WorldSource source, String worldName) { + for (WorldSource.WorldSelector selector : source.getSpec().getWorlds()) { + if (worldName != null && worldName.equals(selector.getName())) { + return selector.getMinecraftVersion(); + } + } + return null; + } + + /** + * Recovers the tenant name from a namespace of the form {@code bluemap-}, the same + * inversion {@code BlueMapMapReconciler.cephUserForNamespace} performs for the identical + * reason: neither {@link WorldSource} nor {@link WorldIngest} carries a direct tenant + * reference, only the namespace they live in, and duplicating this narrow one-line inversion + * is exactly what that class's Javadoc already explains is preferable to a shared helper. + */ + private static String tenantNameForNamespace(String namespace) { + String prefix = "bluemap-"; + return namespace != null && namespace.startsWith(prefix) ? namespace.substring(prefix.length()) : namespace; + } + + private static EnvVar literal(String name, String value) { + return new EnvVarBuilder().withName(name).withValue(value).build(); + } + + /** Credentials must come from a Secret Kubernetes resolves at container start, never inlined. */ + private static EnvVar fromSecret(String name, String secretName, String key) { + return new EnvVarBuilder() + .withName(name) + .withNewValueFrom() + .withNewSecretKeyRef() + .withName(secretName) + .withKey(key) + .endSecretKeyRef() + .endValueFrom() + .build(); + } + + /** Narrow read-only view over whichever of {@code WorldSourceSpec.s3}/{@code .pterodactyl} applies. */ + private record WorldSourceSpecAccess( + String bucket, String endpoint, String prefix, String panelUrl, String serverId, String credentialsSecretName) { + + static WorldSourceSpecAccess s3(WorldSource source) { + var s3 = source.getSpec().getS3(); + return new WorldSourceSpecAccess( + s3.getBucket(), + s3.getEndpoint(), + s3.getPrefix(), + null, + null, + s3.getCredentialsSecretRef().getName()); + } + + static WorldSourceSpecAccess pterodactyl(WorldSource source) { + var pterodactyl = source.getSpec().getPterodactyl(); + return new WorldSourceSpecAccess( + null, + null, + null, + pterodactyl.getPanelUrl(), + pterodactyl.getServerId(), + pterodactyl.getCredentialsSecretRef().getName()); + } + } +} diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/ingest/IngestLogProgress.java b/operator/src/main/java/net/onelitefeather/apus/operator/ingest/IngestLogProgress.java new file mode 100644 index 0000000..89c03a1 --- /dev/null +++ b/operator/src/main/java/net/onelitefeather/apus/operator/ingest/IngestLogProgress.java @@ -0,0 +1,86 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.ingest; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Parses the fine-grained progress {@code WorldIngestReconciler}'s coarse Job/Pod-status polling + * cannot see out of the ingest pod's own log lines -- exactly the fallback {@code + * ingest/README.md}'s "Progress reporting" design note anticipates: {@code IngestMain} prints + * stable, greppable {@code phase=<...>} and {@code progress: NN.N% (done/total bytes)} lines + * instead of running an HTTP server, precisely so a reconciler can read them back out of {@code + * kubectl logs} equivalent. + * + *

Best-effort by construction: every field is {@code null} (or an empty list) when the + * corresponding line was never found, e.g. because the pod has not logged anything yet, or the + * pod could not be reached at all. Callers must treat an all-{@code null} result the same as "no + * new information", never as an error. + */ +public record IngestLogProgress(String phase, Double percent, Long bytesDone, Long bytesTotal, List dimensions) { + + private static final Pattern PHASE = Pattern.compile("phase=([A-Za-z]+)"); + private static final Pattern PROGRESS = Pattern.compile("progress: (\\d+(?:\\.\\d+)?)% \\((\\d+)/(\\d+) bytes\\)"); + private static final Pattern DIMENSIONS = Pattern.compile("dimensions=\\[(.*?)]"); + + /** Scans {@code log} for the last occurrence of each recognised line, in whatever order they appear. */ + public static IngestLogProgress parse(String log) { + if (log == null || log.isBlank()) { + return new IngestLogProgress(null, null, null, null, List.of()); + } + + String phase = lastMatch(PHASE, log, 1); + + Double percent = null; + Long bytesDone = null; + Long bytesTotal = null; + Matcher progressMatcher = PROGRESS.matcher(log); + while (progressMatcher.find()) { + percent = Double.valueOf(progressMatcher.group(1)); + bytesDone = Long.valueOf(progressMatcher.group(2)); + bytesTotal = Long.valueOf(progressMatcher.group(3)); + } + + List dimensions = List.of(); + String dimensionsGroup = lastMatch(DIMENSIONS, log, 1); + if (dimensionsGroup != null && !dimensionsGroup.isBlank()) { + List parsed = new ArrayList<>(); + for (String part : dimensionsGroup.split(",")) { + String trimmed = part.trim(); + if (!trimmed.isEmpty()) { + parsed.add(trimmed); + } + } + dimensions = parsed; + } + + return new IngestLogProgress(phase, percent, bytesDone, bytesTotal, dimensions); + } + + private static String lastMatch(Pattern pattern, String text, int group) { + Matcher matcher = pattern.matcher(text); + String last = null; + while (matcher.find()) { + last = matcher.group(group); + } + return last; + } +} diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/ingest/Secrets.java b/operator/src/main/java/net/onelitefeather/apus/operator/ingest/Secrets.java new file mode 100644 index 0000000..b8cb5b1 --- /dev/null +++ b/operator/src/main/java/net/onelitefeather/apus/operator/ingest/Secrets.java @@ -0,0 +1,59 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.ingest; + +import io.fabric8.kubernetes.api.model.Secret; +import io.fabric8.kubernetes.client.KubernetesClient; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +/** + * Reads a single decoded value out of a {@code Secret}, for the one case in this module that + * needs the actual credential value in-process rather than only a {@code secretKeyRef} (see + * {@code RenderJobBuilder.fromSecret}): {@code WorldSourceReconciler} calling {@code + * WorldSourceConnector.discover()} directly, and {@code WorldIngestReconciler}'s {@code + * AwsBundleStore} pruning retained bundle versions. Both need a real S3/HTTP client built + * in-process; a {@code Job}'s {@code secretKeyRef} only works for a container's own environment. + * + *

Never logged, never written to status. Callers must keep the returned value + * local -- it is a plaintext credential. + */ +public final class Secrets { + + private Secrets() {} + + /** + * Returns the decoded value of {@code key} in the {@code Secret} named {@code secretName} + * in {@code namespace}, or {@code null} if the secret, or the key within it, does not + * exist. + */ + public static String value(KubernetesClient client, String namespace, String secretName, String key) { + if (secretName == null || secretName.isBlank()) { + return null; + } + Secret secret = client.secrets().inNamespace(namespace).withName(secretName).get(); + if (secret == null || secret.getData() == null) { + return null; + } + String encoded = secret.getData().get(key); + if (encoded == null) { + return null; + } + return new String(Base64.getDecoder().decode(encoded), StandardCharsets.UTF_8); + } +} diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/ingest/WorldIngestReconciler.java b/operator/src/main/java/net/onelitefeather/apus/operator/ingest/WorldIngestReconciler.java new file mode 100644 index 0000000..d33ac6a --- /dev/null +++ b/operator/src/main/java/net/onelitefeather/apus/operator/ingest/WorldIngestReconciler.java @@ -0,0 +1,625 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.ingest; + +import io.fabric8.kubernetes.api.model.OwnerReference; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.batch.v1.Job; +import io.fabric8.kubernetes.api.model.batch.v1.JobCondition; +import io.fabric8.kubernetes.api.model.batch.v1.JobStatus; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClientException; +import io.fabric8.kubernetes.client.dsl.NonDeletingOperation; +import io.javaoperatorsdk.operator.api.reconciler.Context; +import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.Reconciler; +import io.javaoperatorsdk.operator.api.reconciler.UpdateControl; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import net.onelitefeather.apus.ingest.BundlePath; +import net.onelitefeather.apus.operator.OperatorConfig; +import net.onelitefeather.apus.operator.api.BlueMapRender; +import net.onelitefeather.apus.operator.api.BundleRef; +import net.onelitefeather.apus.operator.api.Conditions; +import net.onelitefeather.apus.operator.api.Labels; +import net.onelitefeather.apus.operator.api.WorldIngest; +import net.onelitefeather.apus.operator.api.WorldSource; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; + +/** + * Turns a {@link WorldIngest} into a running ingest {@link Job} (via {@link IngestJobBuilder}), + * enforces that only one ingest for a given {@link WorldSource} runs at a time, mirrors the + * job's progress into {@code status}, and on success updates {@code + * WorldSource.status.latestBundle} plus enforces {@code WorldSource.spec.retention}. + * + *

Structurally this is {@code BlueMapRenderReconciler}'s exact shape, applied one level up the + * chain (ingest run -> source, instead of render run -> map): same job-ownership check, same + * optimistic-lock concurrency guard, same "reconciling a terminal ingest is a no-op" rule. See + * that class's Javadoc for the full reasoning; only what differs is called out below. + * + *

Concurrency lock: before creating a Job, this reconciler claims {@code + * WorldSource.status.activeIngest} for its own ingest name via an optimistic {@code + * updateStatus()} -- a 409 Conflict means another ingest claimed it first. If the + * currently-recorded ingest is itself still active (fetched live by name), a second ingest does + * not even attempt the write. + * + *

Source-side bookkeeping happens before the ingest is marked terminal. Updating {@code + * WorldSource.status.latestBundle} and running retention are both separate client calls from the + * {@link UpdateControl} this method returns for the {@link WorldIngest} itself. If the source + * update loses a race (409, someone else concurrently wrote its status), this reconciler leaves + * the ingest's own phase unchanged and reschedules -- not {@code Succeeded} -- so the next + * reconcile retries the source-side write. Marking the ingest {@code Succeeded} first would be a + * mistake: {@link #reconcile} treats a terminal-phase ingest as a permanent no-op, so a lost + * source update after that point would never be retried. + * + *

Retention never deletes a bundle a {@link BlueMapRender} still references. See {@link + * #applyRetention}. + */ +@ControllerConfiguration +public class WorldIngestReconciler implements Reconciler { + + /** Reason set on the {@code Ready} condition while the referenced source does not exist. */ + public static final String SOURCE_NOT_FOUND_REASON = "SourceNotFound"; + + /** Reason set on the {@code Ready} condition while another ingest for the same source is active. */ + public static final String CONCURRENT_INGEST_REASON = "ConcurrentIngestActive"; + + /** Reason set on the {@code Ready} condition when an existing resource fails the ownership check. */ + public static final String RESOURCE_CONFLICT_REASON = "ResourceConflict"; + + /** Reason set once the ingest job has exhausted its retries. */ + public static final String JOB_FAILED_REASON = "JobFailed"; + + /** Reason set on the {@code Ready} condition once the ingest job completed. */ + public static final String SUCCEEDED_REASON = "Succeeded"; + + private static final String PENDING_PHASE = "Pending"; + private static final String EXTRACTING_PHASE = "Extracting"; + private static final String SUCCEEDED_PHASE = "Succeeded"; + private static final String FAILED_PHASE = "Failed"; + + private static final Set TERMINAL_PHASES = Set.of(SUCCEEDED_PHASE, FAILED_PHASE); + + /** Same job->pod label the Kubernetes Job controller always stamps -- see RenderJobBuilder's twin. */ + private static final String JOB_NAME_LABEL = "job-name"; + + private static final Duration RECHECK_INTERVAL = Duration.ofSeconds(10); + + private static final int DEFAULT_KEEP_VERSIONS = 5; + + private final KubernetesClient client; + private final OperatorConfig config; + private final SourceLockClaimer sourceLockClaimer; + private final PodLogFetcher podLogFetcher; + private final BundleStoreFactory bundleStoreFactory; + + public WorldIngestReconciler(KubernetesClient client, OperatorConfig config) { + this( + client, + config, + source -> client.resources(WorldSource.class) + .inNamespace(source.getMetadata().getNamespace()) + .resource(source) + .updateStatus(), + pod -> fetchPodLog(client, pod), + destination -> new AwsBundleStore(software.amazon.awssdk.services.s3.S3Client.builder() + .region(Region.of(destination.region())) + .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create( + Secrets.value( + client, + destination.credentialsNamespace(), + destination.credentialsSecretName(), + "AWS_ACCESS_KEY_ID"), + Secrets.value( + client, + destination.credentialsNamespace(), + destination.credentialsSecretName(), + "AWS_SECRET_ACCESS_KEY")))) + .endpointOverride(java.net.URI.create(destination.endpoint())) + .forcePathStyle(true) + .build())); + } + + /** Test seam: fakes for the source-status lock, pod log fetching and the bundle store. */ + WorldIngestReconciler( + KubernetesClient client, + OperatorConfig config, + SourceLockClaimer sourceLockClaimer, + PodLogFetcher podLogFetcher, + BundleStoreFactory bundleStoreFactory) { + this.client = client; + this.config = config; + this.sourceLockClaimer = sourceLockClaimer; + this.podLogFetcher = podLogFetcher; + this.bundleStoreFactory = bundleStoreFactory; + } + + @Override + public UpdateControl reconcile(WorldIngest ingest, Context context) { + String currentPhase = ingest.getStatus().getPhase(); + if (currentPhase != null && TERMINAL_PHASES.contains(currentPhase)) { + return UpdateControl.noUpdate(); + } + + String namespace = ingest.getMetadata().getNamespace(); + String ingestName = ingest.getMetadata().getName(); + String sourceName = ingest.getSpec().getSourceRef().getName(); + + WorldSource source = (sourceName == null || sourceName.isBlank()) + ? null + : client.resources(WorldSource.class).inNamespace(namespace).withName(sourceName).get(); + if (source == null) { + return pending(ingest, SOURCE_NOT_FOUND_REASON, "source '" + sourceName + "' does not exist"); + } + if (!ownedBySameSource(ingest.getMetadata().getLabels(), source.getMetadata().getName(), source.getMetadata().getUid())) { + return conflict(ingest, "WorldSource", sourceName); + } + + Job existingJob = client.batch().v1().jobs().inNamespace(namespace).withName(ingestName).get(); + if (existingJob != null) { + if (!ownedBySameIngest(existingJob, ingest)) { + return conflict(ingest, "Job", ingestName); + } + return reconcileActiveJob(ingest, source, existingJob, namespace, ingestName); + } + + if (anotherActiveIngestJobExists(namespace, sourceName, ingestName)) { + return pending( + ingest, + CONCURRENT_INGEST_REASON, + "another ingest for source '" + sourceName + "' is already active"); + } + + if (!tryClaimSource(source, namespace, ingestName)) { + return pending( + ingest, + CONCURRENT_INGEST_REASON, + "another ingest for source '" + sourceName + "' is already active"); + } + + Job job = IngestJobBuilder.build(ingest, source, config); + client.batch().v1().jobs().inNamespace(namespace).resource(job).createOr(NonDeletingOperation::update); + + ingest.getStatus().setJobName(ingestName); + ingest.getStatus().setPhase(EXTRACTING_PHASE); + ingest.getStatus().setStartTime(Instant.now().toString()); + Conditions.set( + ingest.getStatus().getConditions(), Conditions.ready(false, EXTRACTING_PHASE, "ingest job submitted")); + return UpdateControl.patchStatus(ingest).rescheduleAfter(RECHECK_INTERVAL); + } + + private UpdateControl reconcileActiveJob( + WorldIngest ingest, WorldSource source, Job job, String namespace, String ingestName) { + Pod pod = findPod(namespace, ingestName); + IngestLogProgress progress = + pod == null ? null : podLogFetcher.fetchLog(pod).map(IngestLogProgress::parse).orElse(null); + + if (isJobSucceeded(job)) { + return onJobSucceeded(ingest, source, namespace, progress); + } + if (isJobFailed(job)) { + ingest.getStatus().setPhase(FAILED_PHASE); + ingest.getStatus().setCompletionTime(Instant.now().toString()); + Conditions.set( + ingest.getStatus().getConditions(), + Conditions.ready(false, JOB_FAILED_REASON, "ingest job failed")); + return UpdateControl.patchStatus(ingest); + } + + if (progress != null) { + applyProgress(ingest, progress); + } + Conditions.set( + ingest.getStatus().getConditions(), + Conditions.ready(false, ingest.getStatus().getPhase(), "ingest job is running")); + return UpdateControl.patchStatus(ingest).rescheduleAfter(RECHECK_INTERVAL); + } + + /** + * Handles a succeeded job: computes the bundle's path/version deterministically (see {@link + * IngestJobBuilder} class Javadoc), writes {@code WorldSource.status.latestBundle} and runs + * retention -- both before marking the ingest itself {@code Succeeded}, so a lost + * race on the source update is retried rather than silently dropped (see class Javadoc). + */ + private UpdateControl onJobSucceeded( + WorldIngest ingest, WorldSource source, String namespace, IngestLogProgress progress) { + String tenant = tenantNameForNamespace(namespace); + String sourceName = source.getMetadata().getName(); + String worldId = ingest.getSpec().getWorldName(); + String version = IngestJobBuilder.bundleVersion(ingest); + String bundlePath = BundlePath.of(tenant, sourceName, worldId, version); + List dimensions = progress == null ? List.of() : progress.dimensions(); + + BundleRef sourceBundle = source.getStatus().getLatestBundle(); + sourceBundle.setPath(bundlePath); + sourceBundle.setVersion(version); + sourceBundle.setDimensions(dimensions); + + try { + sourceLockClaimer.updateStatus(source); + } catch (KubernetesClientException e) { + if (e.getCode() == 409) { + // Someone else wrote WorldSource.status concurrently (e.g. a fresh poll cycle + // updating lastPollTime). Do not mark this ingest terminal yet -- retry the + // whole success path, including this write, on the next reconcile. + return UpdateControl.noUpdate().rescheduleAfter(RECHECK_INTERVAL); + } + throw e; + } + + applyRetention(source, tenant, sourceName, worldId, bundlePath); + + BundleRef ingestBundle = ingest.getStatus().getBundle(); + ingestBundle.setPath(bundlePath); + ingestBundle.setVersion(version); + ingestBundle.setDimensions(dimensions); + ingest.getStatus().setPhase(SUCCEEDED_PHASE); + ingest.getStatus().setCompletionTime(Instant.now().toString()); + Conditions.set( + ingest.getStatus().getConditions(), + Conditions.ready(true, SUCCEEDED_REASON, "ingest completed, bundle at " + bundlePath)); + return UpdateControl.patchStatus(ingest); + } + + /** + * Deletes bundle versions beyond {@code WorldSource.spec.retention.keepVersions}, oldest + * first, but never one still referenced by a {@link BlueMapRender} -- see the class + * Javadoc. A version currently blocked from deletion for that reason is simply skipped; it + * is reconsidered on the next successful ingest's retention pass, once whatever render + * referenced it has moved on or been deleted. + * + *

Best-effort: any failure listing/deleting bundle versions is swallowed rather than + * failing the whole reconciliation -- the ingest itself already succeeded and its own bundle + * is safe; a bucket temporarily unreachable for pruning is not a reason to leave the ingest + * stuck retrying forever. + * + *

Never deletes any {@link WorldSource}'s {@code status.latestBundle}, not just this + * source's own. Bundle listing/deletion is scoped to this source's own prefix ({@link + * BundlePath}, keyed by {@code sourceName}), so a different source's bundles are not even + * visible to this pass -- but that scoping is exactly the property a future change to the + * path scheme could accidentally weaken, and a stray/legacy object under this prefix is not + * inherently impossible either. Checking every source's recorded {@code latestBundle} here + * costs one cheap list call and closes that risk regardless of whether the scoping itself + * stays intact. + */ + private void applyRetention( + WorldSource source, String tenant, String sourceName, String worldId, String justWrittenBundlePath) { + int keepVersions = source.getSpec().getRetention().getKeepVersions(); + if (keepVersions <= 0) { + keepVersions = DEFAULT_KEEP_VERSIONS; + } + try { + BundleStore store = bundleStoreFactory.create(new BundleDestination( + config.bundleS3Region(), config.bundleS3Endpoint(), source.getMetadata().getNamespace(), + config.bundleCredentialsSecretName())); + List versions = new java.util.ArrayList<>( + store.listVersions(tenant, sourceName, worldId, config.bundleBucket())); + versions.sort(java.util.Comparator.comparing(BundleStore.BundleVersion::lastModified).reversed()); + + String namespace = source.getMetadata().getNamespace(); + for (int i = keepVersions; i < versions.size(); i++) { + String version = versions.get(i).version(); + String versionPath = BundlePath.of(tenant, sourceName, worldId, version); + if (versionPath.equals(justWrittenBundlePath)) { + continue; // never prune the version this very run just wrote + } + if (isReferencedByAnyRender(namespace, versionPath)) { + continue; + } + if (isLatestBundleOfAnySource(namespace, versionPath)) { + continue; + } + store.deleteVersion(tenant, sourceName, worldId, version, config.bundleBucket()); + } + } catch (RuntimeException e) { + // See method Javadoc: pruning failures must not block the ingest run itself. + } + } + + /** + * Whether {@code versionPath} is the recorded {@code status.latestBundle.path} of any + * {@link WorldSource} in {@code namespace} -- not just the one this retention pass is running + * for. See {@link #applyRetention}'s Javadoc for why this check exists alongside the + * source-scoped path prefix rather than relying on that scoping alone. + */ + private boolean isLatestBundleOfAnySource(String namespace, String versionPath) { + List sources = + client.resources(WorldSource.class).inNamespace(namespace).list().getItems(); + for (WorldSource candidate : sources) { + String latestPath = candidate.getStatus().getLatestBundle().getPath(); + if (versionPath.equals(latestPath)) { + return true; + } + } + return false; + } + + /** + * Whether any {@link BlueMapRender} in {@code namespace} references bundle version {@code + * versionPath} (the {@code ///} root), regardless of + * that render's own phase. Deliberately not limited to "currently active" renders: a bundle a + * completed render still names in its own spec is history worth keeping intact -- + * over-retaining a bundle costs storage, deleting one a render still names is unrecoverable + * data loss (see the task brief's own framing of this exact risk). + */ + private boolean isReferencedByAnyRender(String namespace, String versionPath) { + List renders = + client.resources(BlueMapRender.class).inNamespace(namespace).list().getItems(); + for (BlueMapRender render : renders) { + String bundleUrl = render.getSpec().getBundleUrl(); + if (bundleUrl != null && referencesBundle(bundleUrl, versionPath)) { + return true; + } + } + return false; + } + + /** + * Boundary-safe substring check: {@code versionPath} must appear as a full path segment in + * {@code bundleUrl}, not merely as a prefix of a longer, different version string (e.g. + * {@code .../v1/...} must not match a URL actually pointing at {@code .../v10/...}). + */ + static boolean referencesBundle(String bundleUrl, String versionPath) { + return bundleUrl.contains("/" + versionPath + "/") || bundleUrl.endsWith("/" + versionPath); + } + + /** + * Mirrors the ingest pod's fine-grained progress into {@code status} -- except a + * terminal phase ({@code Succeeded}/{@code Failed}). {@code IngestMain} logs {@code + * phase=Succeeded} immediately before its process exits, which lands in the pod log strictly + * before the Kubernetes Job controller observes the pod's exit and updates {@code + * status.succeeded} -- a reconcile landing in that window would otherwise copy {@code + * Succeeded} out of the log here, and {@link #reconcile} treats any ingest already in a + * terminal phase as a permanent no-op on every future reconcile. That would skip {@link + * #onJobSucceeded} forever: the bundle the job wrote is never registered on {@code + * WorldSource.status.latestBundle} or {@code WorldIngest.status.bundle}, retention never + * runs, and nothing retries -- a fully-written bundle that is permanently invisible to the + * rest of the system. Terminality belongs exclusively to {@link #isJobSucceeded}/{@link + * #isJobFailed}, evaluated against the Job's own status one line above this method's only + * caller; this method only ever advances the phase to something non-terminal, and only ever + * updates progress numbers. + */ + private static void applyProgress(WorldIngest ingest, IngestLogProgress progress) { + if (progress.phase() != null && !TERMINAL_PHASES.contains(progress.phase())) { + ingest.getStatus().setPhase(progress.phase()); + } + var status = ingest.getStatus().getProgress(); + if (progress.percent() != null) { + status.setPercent(progress.percent()); + } + if (progress.bytesDone() != null) { + status.setBytesDone(progress.bytesDone()); + } + if (progress.bytesTotal() != null) { + status.setBytesTotal(progress.bytesTotal()); + } + } + + private boolean anotherActiveIngestJobExists(String namespace, String sourceName, String excludeIngestName) { + List jobs = client.batch() + .v1() + .jobs() + .inNamespace(namespace) + .withLabel(Labels.SOURCE, sourceName) + .list() + .getItems(); + for (Job job : jobs) { + if (excludeIngestName.equals(job.getMetadata().getName())) { + continue; + } + if (!isJobSucceeded(job) && !isJobFailed(job)) { + return true; + } + } + return false; + } + + /** + * Attempts to claim {@code source.status.activeIngest} for {@code ingestName}. Mirrors + * {@code BlueMapRenderReconciler.tryClaimMap} exactly -- see its Javadoc for the three + * "claimed, proceed" outcomes and the 409-means-lost-the-race handling. + */ + private boolean tryClaimSource(WorldSource source, String namespace, String ingestName) { + String recordedName = source.getStatus().getActiveIngest().getName(); + if (ingestName.equals(recordedName)) { + return true; + } + if (recordedName != null && !recordedName.isBlank() && isSourceIngestStillActive(namespace, recordedName)) { + return false; + } + + source.getStatus().getActiveIngest().setName(ingestName); + source.getStatus().getActiveIngest().setPhase(EXTRACTING_PHASE); + try { + sourceLockClaimer.updateStatus(source); + return true; + } catch (KubernetesClientException e) { + if (e.getCode() == 409) { + return false; + } + throw e; + } + } + + private boolean isSourceIngestStillActive(String namespace, String ingestName) { + WorldIngest recorded = + client.resources(WorldIngest.class).inNamespace(namespace).withName(ingestName).get(); + if (recorded == null) { + return false; + } + String phase = recorded.getStatus().getPhase(); + return phase == null || !TERMINAL_PHASES.contains(phase); + } + + /** + * Best-effort log fetch for the default {@link PodLogFetcher}: a pod that has not started + * yet, or one whose logs are momentarily unreachable, must not fail reconciliation -- exactly + * the same tolerance {@code BlueMapRenderReconciler.HttpProgressFetcher} applies to its own + * best-effort progress source. + */ + private static Optional fetchPodLog(KubernetesClient client, Pod pod) { + try { + String log = client.pods() + .inNamespace(pod.getMetadata().getNamespace()) + .withName(pod.getMetadata().getName()) + .getLog(); + return Optional.ofNullable(log); + } catch (KubernetesClientException e) { + return Optional.empty(); + } + } + + private Pod findPod(String namespace, String ingestName) { + List pods = client.pods() + .inNamespace(namespace) + .withLabel(JOB_NAME_LABEL, ingestName) + .list() + .getItems(); + return pods.isEmpty() ? null : pods.get(0); + } + + private static boolean isJobSucceeded(Job job) { + JobStatus status = job.getStatus(); + if (status == null) { + return false; + } + return (status.getSucceeded() != null && status.getSucceeded() > 0) || hasCondition(status, "Complete"); + } + + /** + * A Job is terminally failed only once its {@code Failed} condition is set -- which the + * Kubernetes Job controller does exactly once, after {@code backoffLimit} retries are + * exhausted. {@code status.failed} (the count of failed pod attempts so far) is deliberately + * not consulted here: {@code backoffLimit} exists precisely so a single transient pod + * failure gets retried, not treated as the whole Job's outcome. Counting pod attempts would + * make this method return {@code true} after the very first failed attempt while the Job + * controller is still going to retry -- {@link #reconcile} would mark the {@link WorldIngest} + * terminally {@code Failed} immediately, yet the Job keeps running underneath it and can still + * write a complete bundle on a later attempt that then has nowhere to be registered, since a + * terminal ingest is never reconciled again. + */ + private static boolean isJobFailed(Job job) { + JobStatus status = job.getStatus(); + if (status == null) { + return false; + } + return hasCondition(status, "Failed"); + } + + private static boolean hasCondition(JobStatus status, String type) { + List conditions = status.getConditions(); + if (conditions == null) { + return false; + } + return conditions.stream().anyMatch(c -> type.equals(c.getType()) && "True".equals(c.getStatus())); + } + + /** + * Checks that {@code ingest}'s own labels name {@code source} by both name and UID -- + * the same owner-check pattern {@code WorldSourceReconciler.ownedBySameSource} applies in the + * opposite direction (there, checking a {@link WorldIngest} it is about to treat as + * already-triggered; here, checking the {@link WorldIngest} being reconciled itself). + * + *

Without this, resolving {@code source} by name alone would let any {@link WorldIngest} -- + * hand-written, or stale after its original source was deleted and a same-named-but-different + * source (a different UID) was created in its place -- read and overwrite that source's + * status and, worse, drive {@link #applyRetention} to delete its bundles. {@code + * WorldSourceReconciler} always stamps {@link Labels#SOURCE}/{@link Labels#SOURCE_UID} on + * every {@link WorldIngest} it creates, so a legitimately-triggered ingest always passes this + * check; one that does not is, by construction, not something this reconciler created the + * lock/retention trust relationship for. + */ + private static boolean ownedBySameSource(Map labels, String sourceName, String sourceUid) { + if (labels == null || sourceUid == null) { + return false; + } + return Objects.equals(sourceName, labels.get(Labels.SOURCE)) && Objects.equals(sourceUid, labels.get(Labels.SOURCE_UID)); + } + + private static boolean ownedBySameIngest(Job job, WorldIngest ingest) { + String ingestUid = ingest.getMetadata().getUid(); + if (ingestUid == null) { + return false; + } + List owners = job.getMetadata().getOwnerReferences(); + if (owners == null) { + return false; + } + return owners.stream() + .anyMatch(ref -> "WorldIngest".equals(ref.getKind()) + && Objects.equals(ingest.getMetadata().getName(), ref.getName()) + && Objects.equals(ingestUid, ref.getUid())); + } + + /** See {@code BlueMapMapReconciler.cephUserForNamespace}/{@code IngestJobBuilder}'s identical inversion. */ + private static String tenantNameForNamespace(String namespace) { + String prefix = "bluemap-"; + return namespace != null && namespace.startsWith(prefix) ? namespace.substring(prefix.length()) : namespace; + } + + private static UpdateControl pending(WorldIngest ingest, String reason, String message) { + ingest.getStatus().setPhase(PENDING_PHASE); + Conditions.set(ingest.getStatus().getConditions(), Conditions.ready(false, reason, message)); + return UpdateControl.patchStatus(ingest).rescheduleAfter(RECHECK_INTERVAL); + } + + private static UpdateControl conflict(WorldIngest ingest, String resourceKind, String resourceName) { + ingest.getStatus().setPhase(PENDING_PHASE); + Conditions.set( + ingest.getStatus().getConditions(), + Conditions.ready( + false, + RESOURCE_CONFLICT_REASON, + "existing " + resourceKind + " '" + resourceName + + "' is not owned by this ingest; refusing to adopt it")); + return UpdateControl.patchStatus(ingest).rescheduleAfter(RECHECK_INTERVAL); + } + + /** + * Performs the optimistic {@code updateStatus()} call {@link #tryClaimSource} and {@link + * #onJobSucceeded} rely on. Exists as a test seam for the same reason {@code + * BlueMapRenderReconciler.MapLockClaimer} does -- the fabric8 mock server used in tests does + * not enforce optimistic concurrency, so a real 409 Conflict cannot be reproduced against it. + */ + @FunctionalInterface + interface SourceLockClaimer { + void updateStatus(WorldSource source); + } + + /** Fetches a pod's full log text, if reachable. */ + @FunctionalInterface + interface PodLogFetcher { + Optional fetchLog(Pod pod); + } + + /** Connection details {@link BundleStoreFactory} needs to build a real {@link BundleStore}. */ + record BundleDestination(String region, String endpoint, String credentialsNamespace, String credentialsSecretName) {} + + /** Builds a {@link BundleStore} for the bundle destination -- a test seam for a fake, in-memory store. */ + @FunctionalInterface + interface BundleStoreFactory { + BundleStore create(BundleDestination destination); + } +} diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/ingest/WorldSourceReconciler.java b/operator/src/main/java/net/onelitefeather/apus/operator/ingest/WorldSourceReconciler.java new file mode 100644 index 0000000..088f1aa --- /dev/null +++ b/operator/src/main/java/net/onelitefeather/apus/operator/ingest/WorldSourceReconciler.java @@ -0,0 +1,396 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.ingest; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.api.model.OwnerReference; +import io.fabric8.kubernetes.api.model.OwnerReferenceBuilder; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.javaoperatorsdk.operator.api.reconciler.Context; +import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.Reconciler; +import io.javaoperatorsdk.operator.api.reconciler.UpdateControl; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeParseException; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import net.onelitefeather.apus.ingest.connector.PterodactylConnector; +import net.onelitefeather.apus.ingest.connector.S3SourceConnector; +import net.onelitefeather.apus.ingest.connector.SourceVersion; +import net.onelitefeather.apus.ingest.connector.WorldSourceConnector; +import net.onelitefeather.apus.operator.api.Conditions; +import net.onelitefeather.apus.operator.api.Labels; +import net.onelitefeather.apus.operator.api.WorldIngest; +import net.onelitefeather.apus.operator.api.WorldSource; + +/** + * Evaluates {@code WorldSource.spec.poll} against {@code status.lastSeenVersion} and, when the + * source reports a version not seen before, creates one {@link WorldIngest} per configured world + * to pull it. + * + *

Only pull sources are pollable. {@code s3} and {@code pterodactyl} report their own + * available versions via {@link WorldSourceConnector#discover}; {@code upload}/{@code push} have + * no connector yet (phase 6 -- see {@code ingest/README.md}) and are always treated as manual + * only, regardless of whether {@code spec.poll} happens to be set. + * + *

{@code discover()} runs here, not inside the ingest Job. This mirrors {@code + * ingest/README.md}'s own "Design notes" section: a single ingest run only ever fetches the one + * version it was told to (deterministic, reproducible), and "is there anything new" is resolved + * exactly once, here, on a schedule. + * + *

Idempotent by construction. The {@link WorldIngest} created for a given (source, + * world, discovered version) triple always gets the same deterministic name ({@link + * #ingestNameFor}); a reconcile that runs again after already having created it for the current + * {@code lastSeenVersion} finds nothing new to do (the version comparison short-circuits before + * ever calling {@link #ingestNameFor}), and even a retry mid-way through creating several worlds' + * ingests just re-attempts the ones it hasn't created yet -- creating an object that already + * exists is a deliberate no-op here, not an error, exactly like {@code createOr(update)} + * elsewhere in this module is idempotent by design. + * + *

Ownership check, mirroring {@code BlueMapMapReconciler}: before this reconciler ever + * treats an existing {@link WorldIngest} of the name it is about to create as "already + * triggered", it checks that resource's labels for one naming this exact source by name + * and UID. A mismatch means the name collided with something unrelated; that world is + * skipped and the source is left with a {@code ResourceConflict} condition instead of silently + * treating a foreign resource as its own ingest run. + */ +@ControllerConfiguration +public class WorldSourceReconciler implements Reconciler { + + /** Reason set on the {@code Ready} condition for a source with no {@code spec.poll} (or an unpollable type). */ + public static final String MANUAL_ONLY_REASON = "ManualOnly"; + + /** Reason set on the {@code Ready} condition when {@code spec.poll} is not a valid Cron expression. */ + public static final String INVALID_POLL_REASON = "InvalidPollExpression"; + + /** Reason set on the {@code Ready} condition when no world is configured to ingest. */ + public static final String NO_WORLDS_CONFIGURED_REASON = "NoWorldsConfigured"; + + /** Reason set on the {@code Ready} condition when the source's connector reported an error. */ + public static final String DISCOVERY_FAILED_REASON = "SourceDiscoveryFailed"; + + /** Reason set on the {@code Ready} condition when the latest discovered version is already ingested. */ + public static final String UP_TO_DATE_REASON = "UpToDate"; + + /** Reason set on the {@code Ready} condition once a new version triggered at least one {@link WorldIngest}. */ + public static final String INGEST_TRIGGERED_REASON = "IngestTriggered"; + + /** Reason set on the {@code Ready} condition when a deterministic ingest name collided with a foreign resource. */ + public static final String RESOURCE_CONFLICT_REASON = "ResourceConflict"; + + private static final Set POLLABLE_TYPES = Set.of("s3", "pterodactyl"); + + private static final String TYPE_S3 = "s3"; + private static final String TYPE_PTERODACTYL = "pterodactyl"; + + /** API group + version the owning {@link WorldSource} is served under. */ + private static final String OWNER_API_VERSION = "bluemap.onelitefeather.net/v1alpha1"; + + private static final String OWNER_KIND = "WorldSource"; + + private final KubernetesClient client; + private final ConnectorResolver connectorResolver; + private final Clock clock; + + public WorldSourceReconciler(KubernetesClient client) { + this(client, WorldSourceReconciler::defaultConnector, Clock.systemUTC()); + } + + /** Test seam: a fake connector/clock make discovery and cron due-ness deterministic. */ + WorldSourceReconciler(KubernetesClient client, ConnectorResolver connectorResolver, Clock clock) { + this.client = client; + this.connectorResolver = connectorResolver; + this.clock = clock; + } + + @Override + public UpdateControl reconcile(WorldSource source, Context context) { + String type = source.getSpec().getType(); + String poll = source.getSpec().getPoll(); + + if (poll == null || poll.isBlank() || !POLLABLE_TYPES.contains(type)) { + return manualOnly(source); + } + + CronSchedule schedule; + try { + schedule = CronSchedule.parse(poll); + } catch (CronSchedule.InvalidCronExpressionException e) { + return terminalCondition(source, INVALID_POLL_REASON, e.getMessage()); + } + + ZonedDateTime now = ZonedDateTime.now(clock); + ZonedDateTime lastPoll = parseInstant(source.getStatus().getLastPollTime()); + if (!schedule.isDue(lastPoll, now)) { + return UpdateControl.noUpdate().rescheduleAfter(schedule.timeToNext(now)); + } + + if (source.getSpec().getWorlds().isEmpty()) { + return pending(source, NO_WORLDS_CONFIGURED_REASON, "no worlds configured to ingest", schedule, now); + } + + WorldSourceConnector connector = connectorResolver.resolve(type); + Map config = sourceConfig(source); + + List versions; + try { + versions = connector.discover(config); + } catch (RuntimeException e) { + // Never let a connection/credential problem surface with its raw message here -- + // it may embed request details; discover() implementations already keep secrets + // out of exception messages, but this is the one place in the operator that talks + // to an external source directly, so the boundary is defended explicitly too. + return pending( + source, + DISCOVERY_FAILED_REASON, + "failed to list versions at the source: " + e.getClass().getSimpleName(), + schedule, + now); + } + + source.getStatus().setLastPollTime(now.toInstant().toString()); + + Optional latest = versions.stream().max(Comparator.comparing(SourceVersion::createdAt)); + if (latest.isEmpty()) { + Conditions.set( + source.getStatus().getConditions(), + Conditions.ready(true, UP_TO_DATE_REASON, "no versions available at the source yet")); + return UpdateControl.patchStatus(source).rescheduleAfter(schedule.timeToNext(now)); + } + + String latestId = latest.get().id(); + if (latestId.equals(source.getStatus().getLastSeenVersion())) { + Conditions.set( + source.getStatus().getConditions(), + Conditions.ready(true, UP_TO_DATE_REASON, "already ingested version '" + latestId + "'")); + return UpdateControl.patchStatus(source).rescheduleAfter(schedule.timeToNext(now)); + } + + boolean conflict = triggerIngests(source, latestId); + source.getStatus().setLastSeenVersion(latestId); + Conditions.set( + source.getStatus().getConditions(), + conflict + ? Conditions.ready( + false, + RESOURCE_CONFLICT_REASON, + "an ingest name for version '" + latestId + + "' collided with a resource not owned by this source") + : Conditions.ready(true, INGEST_TRIGGERED_REASON, "triggered ingest for version '" + latestId + "'")); + return UpdateControl.patchStatus(source).rescheduleAfter(schedule.timeToNext(now)); + } + + /** + * Creates one {@link WorldIngest} per configured world for {@code latestVersion}, skipping + * any that already exist (idempotent retry) and any name collision with a foreign resource. + * + * @return {@code true} if at least one collision with a foreign resource was found + */ + private boolean triggerIngests(WorldSource source, String latestVersion) { + String namespace = source.getMetadata().getNamespace(); + String sourceName = source.getMetadata().getName(); + String sourceUid = source.getMetadata().getUid(); + + boolean conflict = false; + for (WorldSource.WorldSelector selector : source.getSpec().getWorlds()) { + String ingestName = ingestNameFor(sourceName, selector.getName(), latestVersion); + WorldIngest existing = + client.resources(WorldIngest.class).inNamespace(namespace).withName(ingestName).get(); + if (existing != null) { + if (!ownedBySameSource(existing.getMetadata().getLabels(), sourceName, sourceUid)) { + conflict = true; + } + continue; // already triggered for this version -- idempotent no-op either way + } + + WorldIngest ingest = new WorldIngest(); + ingest.setMetadata(new ObjectMetaBuilder() + .withName(ingestName) + .withNamespace(namespace) + .withLabels(ingestLabels(sourceName, sourceUid)) + .withOwnerReferences(ownerReference(source)) + .build()); + ingest.getSpec().getSourceRef().setName(sourceName); + ingest.getSpec().setSourceVersion(latestVersion); + ingest.getSpec().setWorldName(selector.getName()); + client.resources(WorldIngest.class).inNamespace(namespace).resource(ingest).create(); + } + return conflict; + } + + /** + * Builds the connector configuration map for {@code source.spec.type}, resolving the + * referenced credentials Secret (if any) to its decoded value -- never logged, never + * written to status; see {@link Secrets}. + */ + private Map sourceConfig(WorldSource source) { + String namespace = source.getMetadata().getNamespace(); + Map config = new LinkedHashMap<>(); + if (TYPE_S3.equals(source.getSpec().getType())) { + var s3 = source.getSpec().getS3(); + putIfPresent(config, S3SourceConnector.CONFIG_BUCKET, s3.getBucket()); + putIfPresent(config, S3SourceConnector.CONFIG_ENDPOINT, s3.getEndpoint()); + putIfPresent(config, S3SourceConnector.CONFIG_PREFIX, s3.getPrefix()); + String secretName = s3.getCredentialsSecretRef().getName(); + putIfPresent( + config, + S3SourceConnector.CONFIG_ACCESS_KEY_ID, + Secrets.value(client, namespace, secretName, "AWS_ACCESS_KEY_ID")); + putIfPresent( + config, + S3SourceConnector.CONFIG_SECRET_ACCESS_KEY, + Secrets.value(client, namespace, secretName, "AWS_SECRET_ACCESS_KEY")); + } else if (TYPE_PTERODACTYL.equals(source.getSpec().getType())) { + var pterodactyl = source.getSpec().getPterodactyl(); + putIfPresent(config, PterodactylConnector.CONFIG_PANEL_URL, pterodactyl.getPanelUrl()); + putIfPresent(config, PterodactylConnector.CONFIG_SERVER_ID, pterodactyl.getServerId()); + String secretName = pterodactyl.getCredentialsSecretRef().getName(); + putIfPresent( + config, PterodactylConnector.CONFIG_API_KEY, Secrets.value(client, namespace, secretName, "API_KEY")); + } + return config; + } + + private static void putIfPresent(Map config, String key, String value) { + if (value != null && !value.isBlank()) { + config.put(key, value); + } + } + + private static WorldSourceConnector defaultConnector(String type) { + return switch (type) { + case TYPE_S3 -> new S3SourceConnector(); + case TYPE_PTERODACTYL -> new PterodactylConnector(); + default -> throw new IllegalStateException("unsupported source type: " + type); + }; + } + + /** + * Deterministic name for the {@link WorldIngest} triggered for one (source, world, source + * version) triple -- what makes {@link #triggerIngests} idempotent. Kubernetes resource + * names must be valid RFC 1123 DNS subdomain labels (lowercase alphanumeric and {@code -}, + * max 253 characters); a raw source version id (an S3 key, a Pterodactyl backup UUID, ...) + * is not guaranteed to satisfy that, so it is never used verbatim. Instead: a sanitised, + * truncated version of it, plus a short hash of the *original, unsanitised* id so that two + * different version ids which happen to sanitise to the same string (e.g. differing only in + * characters {@link #sanitize} strips) still get different, non-colliding names. + */ + static String ingestNameFor(String sourceName, String worldName, String version) { + String hash = shortHash(version); + String base = sanitize(sourceName) + "-" + sanitize(worldName) + "-" + sanitize(version); + int maxBaseLength = 253 - 1 - hash.length(); + if (base.length() > maxBaseLength) { + base = base.substring(0, maxBaseLength); + } + return base + "-" + hash; + } + + private static String sanitize(String value) { + String lowered = value == null ? "" : value.toLowerCase(java.util.Locale.ROOT); + String replaced = lowered.replaceAll("[^a-z0-9]+", "-"); + String trimmed = replaced.replaceAll("^-+|-+$", ""); + return trimmed.isEmpty() ? "x" : trimmed; + } + + private static String shortHash(String value) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] hash = digest.digest(value.getBytes(StandardCharsets.UTF_8)); + StringBuilder hex = new StringBuilder(8); + for (int i = 0; i < 4; i++) { + hex.append(String.format("%02x", hash[i])); + } + return hex.toString(); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is not available", e); + } + } + + private static Map ingestLabels(String sourceName, String sourceUid) { + Map labels = Labels.standard("world-ingest", sourceName); + labels.put(Labels.SOURCE, sourceName); + if (sourceUid != null && !sourceUid.isBlank()) { + labels.put(Labels.SOURCE_UID, sourceUid); + } + return labels; + } + + private static boolean ownedBySameSource(Map labels, String sourceName, String sourceUid) { + if (labels == null || sourceUid == null) { + return false; + } + return Objects.equals(sourceName, labels.get(Labels.SOURCE)) && Objects.equals(sourceUid, labels.get(Labels.SOURCE_UID)); + } + + private static OwnerReference ownerReference(WorldSource source) { + return new OwnerReferenceBuilder() + .withApiVersion(OWNER_API_VERSION) + .withKind(OWNER_KIND) + .withName(source.getMetadata().getName()) + .withUid(source.getMetadata().getUid()) + .withController(true) + .withBlockOwnerDeletion(true) + .build(); + } + + private static ZonedDateTime parseInstant(String value) { + if (value == null || value.isBlank()) { + return null; + } + try { + return Instant.parse(value).atZone(ZoneOffset.UTC); + } catch (DateTimeParseException e) { + return null; + } + } + + private static UpdateControl manualOnly(WorldSource source) { + Conditions.set( + source.getStatus().getConditions(), + Conditions.ready(true, MANUAL_ONLY_REASON, "no automatic poll configured for this source")); + return UpdateControl.patchStatus(source); + } + + private static UpdateControl terminalCondition(WorldSource source, String reason, String message) { + Conditions.set(source.getStatus().getConditions(), Conditions.ready(false, reason, message)); + return UpdateControl.patchStatus(source); + } + + private static UpdateControl pending( + WorldSource source, String reason, String message, CronSchedule schedule, ZonedDateTime now) { + Conditions.set(source.getStatus().getConditions(), Conditions.ready(false, reason, message)); + return UpdateControl.patchStatus(source).rescheduleAfter(schedule.timeToNext(now)); + } + + /** Resolves the {@link WorldSourceConnector} implementation for a {@code spec.type} value. */ + @FunctionalInterface + interface ConnectorResolver { + WorldSourceConnector resolve(String type); + } +} diff --git a/operator/src/main/java/net/onelitefeather/apus/operator/render/BlueMapRenderReconciler.java b/operator/src/main/java/net/onelitefeather/apus/operator/render/BlueMapRenderReconciler.java index 58e1611..0769e87 100644 --- a/operator/src/main/java/net/onelitefeather/apus/operator/render/BlueMapRenderReconciler.java +++ b/operator/src/main/java/net/onelitefeather/apus/operator/render/BlueMapRenderReconciler.java @@ -443,12 +443,24 @@ private static boolean isJobSucceeded(Job job) { return (status.getSucceeded() != null && status.getSucceeded() > 0) || hasCondition(status, "Complete"); } + /** + * A Job is terminally failed only once its {@code Failed} condition is set -- which the + * Kubernetes Job controller does exactly once, after {@code backoffLimit} retries are + * exhausted. {@code status.failed} (the count of failed pod attempts so far) is deliberately + * not consulted here: {@code backoffLimit} exists precisely so a single transient pod + * failure gets retried, not treated as the whole Job's outcome. Counting pod attempts would + * make this method return {@code true} after the very first failed attempt while the Job + * controller is still going to retry -- {@link #reconcileActiveJob} would mark the {@link + * BlueMapRender} terminally {@code Failed} immediately, even though the Job keeps running + * underneath it and may still succeed on a later attempt. See {@code WorldIngestReconciler}'s + * identical fix -- same underlying mistake, found in both reconcilers. + */ private static boolean isJobFailed(Job job) { JobStatus status = job.getStatus(); if (status == null) { return false; } - return (status.getFailed() != null && status.getFailed() > 0) || hasCondition(status, "Failed"); + return hasCondition(status, "Failed"); } private static boolean hasCondition(JobStatus status, String type) { diff --git a/operator/src/test/java/net/onelitefeather/apus/operator/ApusOperatorTest.java b/operator/src/test/java/net/onelitefeather/apus/operator/ApusOperatorTest.java index 290338d..7f24b97 100644 --- a/operator/src/test/java/net/onelitefeather/apus/operator/ApusOperatorTest.java +++ b/operator/src/test/java/net/onelitefeather/apus/operator/ApusOperatorTest.java @@ -25,6 +25,8 @@ import io.javaoperatorsdk.operator.Operator; import java.util.Set; import java.util.stream.Collectors; +import net.onelitefeather.apus.operator.ingest.WorldIngestReconciler; +import net.onelitefeather.apus.operator.ingest.WorldSourceReconciler; import net.onelitefeather.apus.operator.map.BlueMapMapReconciler; import net.onelitefeather.apus.operator.render.BlueMapRenderReconciler; import net.onelitefeather.apus.operator.tenant.TenantReconciler; @@ -48,17 +50,19 @@ class ApusOperatorTest { KubernetesClient client; @Test - void registersAllThreeReconcilers() { + void registersAllFiveReconcilers() { Operator operator = new Operator(o -> o.withKubernetesClient(client)); ApusOperator.registerReconcilers(operator, client, OperatorConfig.defaults()); - assertEquals(3, operator.getRegisteredControllersNumber()); + assertEquals(5, operator.getRegisteredControllersNumber()); Set reconcilerClassNames = operator.getRegisteredControllers().stream() .map(controller -> controller.getConfiguration().getAssociatedReconcilerClassName()) .collect(Collectors.toSet()); assertTrue(reconcilerClassNames.contains(TenantReconciler.class.getName())); assertTrue(reconcilerClassNames.contains(BlueMapMapReconciler.class.getName())); assertTrue(reconcilerClassNames.contains(BlueMapRenderReconciler.class.getName())); + assertTrue(reconcilerClassNames.contains(WorldSourceReconciler.class.getName())); + assertTrue(reconcilerClassNames.contains(WorldIngestReconciler.class.getName())); } } diff --git a/operator/src/test/java/net/onelitefeather/apus/operator/CrdGenerationTest.java b/operator/src/test/java/net/onelitefeather/apus/operator/CrdGenerationTest.java index 0504fa6..31f3ce3 100644 --- a/operator/src/test/java/net/onelitefeather/apus/operator/CrdGenerationTest.java +++ b/operator/src/test/java/net/onelitefeather/apus/operator/CrdGenerationTest.java @@ -169,4 +169,37 @@ void blueMapRenderIsNamespaceScoped() { assertEquals("Namespaced", crd.getSpec().getScope(), "BlueMapRender must be namespace-scoped"); } + + @Test + void generatesTheWorldSourceCrdWithExpectedIdentity() { + CustomResourceDefinition crd = loadCrd("worldsources.bluemap.onelitefeather.net-v1.yml"); + + assertEquals("bluemap.onelitefeather.net", crd.getSpec().getGroup()); + assertEquals("WorldSource", crd.getSpec().getNames().getKind()); + assertEquals("worldsources", crd.getSpec().getNames().getPlural()); + } + + @Test + void worldSourceIsNamespaceScoped() { + CustomResourceDefinition crd = loadCrd("worldsources.bluemap.onelitefeather.net-v1.yml"); + + // A source belongs to exactly one tenant's namespace, exactly like BlueMapMap. + assertEquals("Namespaced", crd.getSpec().getScope(), "WorldSource must be namespace-scoped"); + } + + @Test + void generatesTheWorldIngestCrdWithExpectedIdentity() { + CustomResourceDefinition crd = loadCrd("worldingests.bluemap.onelitefeather.net-v1.yml"); + + assertEquals("bluemap.onelitefeather.net", crd.getSpec().getGroup()); + assertEquals("WorldIngest", crd.getSpec().getNames().getKind()); + assertEquals("worldingests", crd.getSpec().getNames().getPlural()); + } + + @Test + void worldIngestIsNamespaceScoped() { + CustomResourceDefinition crd = loadCrd("worldingests.bluemap.onelitefeather.net-v1.yml"); + + assertEquals("Namespaced", crd.getSpec().getScope(), "WorldIngest must be namespace-scoped"); + } } diff --git a/operator/src/test/java/net/onelitefeather/apus/operator/OperatorConfigTest.java b/operator/src/test/java/net/onelitefeather/apus/operator/OperatorConfigTest.java index 49cb122..b8edb6b 100644 --- a/operator/src/test/java/net/onelitefeather/apus/operator/OperatorConfigTest.java +++ b/operator/src/test/java/net/onelitefeather/apus/operator/OperatorConfigTest.java @@ -32,6 +32,10 @@ void defaultsMatchTheFeatherCoreCluster() { assertEquals("feather-s3", config.cephObjectStore()); assertEquals("ceph-bucket-fr01", config.bucketStorageClass()); assertEquals("apus/runner:dev", config.runnerImage()); + assertEquals("apus/ingest:dev", config.ingestImage()); + assertEquals("apus-bundles", config.bundleBucket()); + assertEquals("us-east-1", config.bundleS3Region()); + assertEquals("apus-bundle-credentials", config.bundleCredentialsSecretName()); } @Test @@ -49,12 +53,17 @@ void fromEnvironmentFallsBackToDefaultsWhenBlank() { } @Test - void fromEnvironmentReadsAllFourVariables() { - Map env = Map.of( - "APUS_ROOK_NAMESPACE", "rook-ceph-de01", - "APUS_CEPH_OBJECT_STORE", "feather-s3-de", - "APUS_BUCKET_STORAGE_CLASS", "ceph-bucket-de01", - "APUS_RUNNER_IMAGE", "apus/runner:1.2.3"); + void fromEnvironmentReadsAllVariables() { + Map env = Map.ofEntries( + Map.entry("APUS_ROOK_NAMESPACE", "rook-ceph-de01"), + Map.entry("APUS_CEPH_OBJECT_STORE", "feather-s3-de"), + Map.entry("APUS_BUCKET_STORAGE_CLASS", "ceph-bucket-de01"), + Map.entry("APUS_RUNNER_IMAGE", "apus/runner:1.2.3"), + Map.entry("APUS_INGEST_IMAGE", "apus/ingest:1.2.3"), + Map.entry("APUS_BUNDLE_BUCKET", "bundles-de"), + Map.entry("APUS_BUNDLE_S3_ENDPOINT", "http://rgw.de.svc:80"), + Map.entry("APUS_BUNDLE_S3_REGION", "eu-central-1"), + Map.entry("APUS_BUNDLE_CREDENTIALS_SECRET", "bundle-creds-de")); OperatorConfig config = OperatorConfig.fromEnvironment(env::get); @@ -62,5 +71,10 @@ void fromEnvironmentReadsAllFourVariables() { assertEquals("feather-s3-de", config.cephObjectStore()); assertEquals("ceph-bucket-de01", config.bucketStorageClass()); assertEquals("apus/runner:1.2.3", config.runnerImage()); + assertEquals("apus/ingest:1.2.3", config.ingestImage()); + assertEquals("bundles-de", config.bundleBucket()); + assertEquals("http://rgw.de.svc:80", config.bundleS3Endpoint()); + assertEquals("eu-central-1", config.bundleS3Region()); + assertEquals("bundle-creds-de", config.bundleCredentialsSecretName()); } } diff --git a/operator/src/test/java/net/onelitefeather/apus/operator/api/IngestResourceTest.java b/operator/src/test/java/net/onelitefeather/apus/operator/api/IngestResourceTest.java new file mode 100644 index 0000000..108ac27 --- /dev/null +++ b/operator/src/test/java/net/onelitefeather/apus/operator/api/IngestResourceTest.java @@ -0,0 +1,116 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.fabric8.kubernetes.api.model.Namespaced; +import java.lang.reflect.Field; +import org.junit.jupiter.api.Test; + +class IngestResourceTest { + + @Test + void bothResourcesAreNamespaced() { + assertTrue(Namespaced.class.isAssignableFrom(WorldSource.class)); + assertTrue(Namespaced.class.isAssignableFrom(WorldIngest.class)); + } + + @Test + void specGroupsAreInitialisedSoReconcilersNeverSeeNull() { + WorldSource source = new WorldSource(); + assertNotNull(source.getSpec().getS3()); + assertNotNull(source.getSpec().getPterodactyl()); + assertNotNull(source.getSpec().getWorlds()); + assertNotNull(source.getSpec().getRetention()); + assertNotNull(source.getStatus().getLatestBundle()); + + WorldIngest ingest = new WorldIngest(); + assertNotNull(ingest.getSpec().getSourceRef()); + assertNotNull(ingest.getStatus().getProgress()); + assertNotNull(ingest.getStatus().getBundle()); + } + + @Test + void retentionDefaultsToFiveVersions() { + assertEquals(5, new WorldSource().getSpec().getRetention().getKeepVersions()); + } + + @Test + void layoutDefaultsToAutomaticDetection() { + WorldSource.WorldSelector selector = new WorldSource.WorldSelector(); + assertEquals("auto", selector.getLayout()); + } + + @Test + void everyResourceHasANonNullSpecAndStatusRightAfterConstruction() { + // CustomResource's default initSpec()/initStatus() return null; a subclass has to + // override both or `new X().getSpec()` is null until something (e.g. Jackson + // deserialisation from the API server) overwrites the field. This trap already blocked + // three parallel Phase 2a tasks once -- checked explicitly here so it cannot repeat. + assertNotNull(new WorldSource().getSpec(), "WorldSource.getSpec() must not be null right after construction"); + assertNotNull( + new WorldSource().getStatus(), "WorldSource.getStatus() must not be null right after construction"); + assertNotNull( + new WorldIngest().getSpec(), "WorldIngest.getSpec() must not be null right after construction"); + assertNotNull( + new WorldIngest().getStatus(), "WorldIngest.getStatus() must not be null right after construction"); + } + + @Test + void allNestedGroupsAreInitialisedRecursivelyForBothResources() { + // A check that only looks at the first level of a spec/status (as + // specGroupsAreInitialisedSoReconcilersNeverSeeNull() above does) can miss a group + // nested two levels deep, e.g. S3Source.credentialsSecretRef. Walking every nested + // Apus-owned group recursively closes that gap for the current fields and for any + // added later -- see ApusResourceTest for the identical check on the Phase 2a + // resources, and the Phase 2a incident that motivated it. + assertNoUninitialisedGroup(new WorldSource().getSpec()); + assertNoUninitialisedGroup(new WorldSource().getStatus()); + assertNoUninitialisedGroup(new WorldIngest().getSpec()); + assertNoUninitialisedGroup(new WorldIngest().getStatus()); + } + + private static void assertNoUninitialisedGroup(Object group) { + for (Field field : group.getClass().getDeclaredFields()) { + if (field.isSynthetic() || !isApusOwnedType(field.getType())) { + continue; + } + field.setAccessible(true); + Object value; + try { + value = field.get(group); + } catch (IllegalAccessException e) { + throw new AssertionError("could not read field " + field, e); + } + assertNotNull( + value, + group.getClass().getSimpleName() + "." + field.getName() + + " must be initialised in its field declaration, not left null"); + assertNoUninitialisedGroup(value); + } + } + + private static boolean isApusOwnedType(Class type) { + // Nested static classes (e.g. WorldSourceSpec.S3Source) still report their enclosing + // top-level class's package, so a plain equality check also covers them. + return "net.onelitefeather.apus.operator.api".equals(type.getPackageName()); + } +} diff --git a/operator/src/test/java/net/onelitefeather/apus/operator/ingest/CronScheduleTest.java b/operator/src/test/java/net/onelitefeather/apus/operator/ingest/CronScheduleTest.java new file mode 100644 index 0000000..d11e12d --- /dev/null +++ b/operator/src/test/java/net/onelitefeather/apus/operator/ingest/CronScheduleTest.java @@ -0,0 +1,81 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.ingest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import org.junit.jupiter.api.Test; + +class CronScheduleTest { + + private static final ZonedDateTime NOON = ZonedDateTime.of(2026, 8, 9, 12, 0, 0, 0, ZoneOffset.UTC); + + @Test + void neverPolledBeforeIsAlwaysDue() { + CronSchedule schedule = CronSchedule.parse("0 * * * *"); // hourly + + assertTrue(schedule.isDue(null, NOON), "a source that has never been polled must poll immediately"); + } + + @Test + void notDueBeforeTheNextScheduledFireTime() { + CronSchedule schedule = CronSchedule.parse("0 * * * *"); // hourly, fires on the hour + + ZonedDateTime lastPoll = NOON; // just polled at 12:00 + ZonedDateTime fiveMinutesLater = NOON.plusMinutes(5); + + assertFalse(schedule.isDue(lastPoll, fiveMinutesLater), "next hourly fire is at 13:00, not yet reached"); + } + + @Test + void dueOnceTheNextScheduledFireTimeHasPassed() { + CronSchedule schedule = CronSchedule.parse("0 * * * *"); // hourly + + ZonedDateTime lastPoll = NOON; + ZonedDateTime oneHourLater = NOON.plusHours(1).plusSeconds(1); + + assertTrue(schedule.isDue(lastPoll, oneHourLater), "the 13:00 fire has already passed"); + } + + @Test + void timeToNextReflectsTheRemainingWaitUntilTheNextFire() { + CronSchedule schedule = CronSchedule.parse("0 * * * *"); // hourly, fires on the hour + + Duration remaining = schedule.timeToNext(NOON.plusMinutes(45)); + + assertEquals(Duration.ofMinutes(15), remaining); + } + + @Test + void rejectsAnInvalidExpressionInsteadOfGuessingAMeaning() { + assertThrows(CronSchedule.InvalidCronExpressionException.class, () -> CronSchedule.parse("not a cron")); + } + + @Test + void rejectsASixFieldQuartzStyleExpression() { + // This module deliberately speaks five-field Unix cron only (matching + // Kubernetes CronJob.spec.schedule) -- a seconds field must not be silently accepted. + assertThrows(CronSchedule.InvalidCronExpressionException.class, () -> CronSchedule.parse("0 0 * * * *")); + } +} diff --git a/operator/src/test/java/net/onelitefeather/apus/operator/ingest/IngestJobBuilderTest.java b/operator/src/test/java/net/onelitefeather/apus/operator/ingest/IngestJobBuilderTest.java new file mode 100644 index 0000000..b75a572 --- /dev/null +++ b/operator/src/test/java/net/onelitefeather/apus/operator/ingest/IngestJobBuilderTest.java @@ -0,0 +1,255 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.ingest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import io.fabric8.kubernetes.api.model.EnvVar; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.api.model.OwnerReference; +import io.fabric8.kubernetes.api.model.batch.v1.Job; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; +import net.onelitefeather.apus.operator.OperatorConfig; +import net.onelitefeather.apus.operator.api.Labels; +import net.onelitefeather.apus.operator.api.WorldIngest; +import net.onelitefeather.apus.operator.api.WorldSource; +import org.junit.jupiter.api.Test; + +class IngestJobBuilderTest { + + private WorldSource s3Source(String name) { + WorldSource source = new WorldSource(); + source.setMetadata(new ObjectMetaBuilder() + .withName(name) + .withNamespace("bluemap-friends") + .withUid(UUID.randomUUID().toString()) + .build()); + source.getSpec().setType("s3"); + source.getSpec().getS3().setBucket("backups"); + source.getSpec().getS3().setPrefix("survival/"); + source.getSpec().getS3().setEndpoint("http://minio.example.svc:9000"); + source.getSpec().getS3().getCredentialsSecretRef().setName("source-creds"); + WorldSource.WorldSelector selector = new WorldSource.WorldSelector(); + selector.setName("world"); + selector.setLayout("bukkit"); + source.getSpec().getWorlds().add(selector); + return source; + } + + private WorldSource pterodactylSource(String name) { + WorldSource source = new WorldSource(); + source.setMetadata(new ObjectMetaBuilder() + .withName(name) + .withNamespace("bluemap-friends") + .withUid(UUID.randomUUID().toString()) + .build()); + source.getSpec().setType("pterodactyl"); + source.getSpec().getPterodactyl().setPanelUrl("https://panel.example.com"); + source.getSpec().getPterodactyl().setServerId("abc123"); + source.getSpec().getPterodactyl().getCredentialsSecretRef().setName("panel-creds"); + return source; + } + + private WorldIngest ingest(String name, String sourceVersion) { + WorldIngest ingest = new WorldIngest(); + ingest.setMetadata(new ObjectMetaBuilder() + .withName(name) + .withNamespace("bluemap-friends") + .withUid(UUID.randomUUID().toString()) + .build()); + ingest.getSpec().getSourceRef().setName("survival-source"); + ingest.getSpec().setSourceVersion(sourceVersion); + ingest.getSpec().setWorldName("world"); + return ingest; + } + + private Map envOf(Job job) { + List env = + job.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv(); + return env.stream().collect(Collectors.toMap(EnvVar::getName, Function.identity())); + } + + @Test + void suppliesEveryMandatoryEnvironmentVariableForAnS3Source() { + WorldIngest ingest = ingest("survival-source-world-v1", "2026-08-01T00-00-00Z.zip"); + Job job = IngestJobBuilder.build(ingest, s3Source("survival-source"), OperatorConfig.defaults()); + + Map env = envOf(job); + + for (String required : List.of( + "APUS_SOURCE_TYPE", + "APUS_WORLD_NAME", + "APUS_SOURCE_VERSION", + "APUS_BUNDLE_BUCKET", + "APUS_BUNDLE_TENANT", + "APUS_BUNDLE_SOURCE_NAME", + "APUS_BUNDLE_WORLD_ID", + "APUS_BUNDLE_VERSION", + "APUS_S3_ENDPOINT", + "APUS_S3" + "_ACCESS_KEY", + "APUS_S3" + "_SECRET_KEY", + "APUS_SOURCE_S3_BUCKET")) { + assertNotNull(env.get(required), "missing mandatory variable " + required); + } + + assertEquals("s3", env.get("APUS_SOURCE_TYPE").getValue()); + assertEquals("world", env.get("APUS_WORLD_NAME").getValue()); + assertEquals("2026-08-01T00-00-00Z.zip", env.get("APUS_SOURCE_VERSION").getValue()); + assertEquals("friends", env.get("APUS_BUNDLE_TENANT").getValue(), "tenant recovered from the namespace"); + assertEquals( + "survival-source", + env.get("APUS_BUNDLE_SOURCE_NAME").getValue(), + "bundle path must be scoped by the owning source's name (see BundlePath)"); + assertEquals("world", env.get("APUS_BUNDLE_WORLD_ID").getValue()); + assertEquals( + "survival-source-world-v1", + env.get("APUS_BUNDLE_VERSION").getValue(), + "bundle version is the WorldIngest's own name, distinct from the source version"); + assertEquals("backups", env.get("APUS_SOURCE_S3_BUCKET").getValue()); + assertEquals("survival/", env.get("APUS_SOURCE_S3_PREFIX").getValue()); + assertEquals("http://minio.example.svc:9000", env.get("APUS_SOURCE_S3_ENDPOINT").getValue()); + assertEquals("bukkit", env.get("APUS_LAYOUT").getValue(), "layout comes from the matching WorldSelector"); + } + + @Test + void mcVersionIsOmittedWhenTheMatchingWorldSelectorDoesNotConfigureOne() { + Job job = IngestJobBuilder.build( + ingest("i1", "v1"), s3Source("survival-source"), OperatorConfig.defaults()); + + assertNull(envOf(job).get("APUS_MC_VERSION"), "no minecraftVersion configured on the WorldSelector"); + } + + @Test + void mcVersionIsPassedThroughFromTheMatchingWorldSelector() { + WorldSource source = s3Source("survival-source"); + source.getSpec().getWorlds().get(0).setMinecraftVersion("1.21.10"); + + Job job = IngestJobBuilder.build(ingest("i1", "v1"), source, OperatorConfig.defaults()); + + assertEquals("1.21.10", envOf(job).get("APUS_MC_VERSION").getValue()); + } + + @Test + void theIngestContainerHasAnEphemeralStorageRequestAndLimit() { + Job job = IngestJobBuilder.build( + ingest("i1", "v1"), s3Source("survival-source"), OperatorConfig.defaults()); + + var resources = job.getSpec().getTemplate().getSpec().getContainers().get(0).getResources(); + assertNotNull(resources.getRequests().get("ephemeral-storage"), "no volume is mounted for the work directory"); + assertNotNull(resources.getLimits().get("ephemeral-storage")); + } + + @Test + void bundleCredentialsComeFromASecretReferenceNeverALiteralValue() { + Job job = IngestJobBuilder.build( + ingest("i1", "v1"), s3Source("survival-source"), OperatorConfig.defaults()); + EnvVar accessKey = envOf(job).get("APUS_S3" + "_ACCESS_KEY"); + + assertNull(accessKey.getValue(), "must never inline the credential"); + assertNotNull(accessKey.getValueFrom().getSecretKeyRef()); + assertEquals("apus-bundle-credentials", accessKey.getValueFrom().getSecretKeyRef().getName()); + assertEquals("AWS_ACCESS_KEY_ID", accessKey.getValueFrom().getSecretKeyRef().getKey()); + } + + @Test + void sourceCredentialsAreOmittedWhenNoSecretIsReferenced() { + WorldSource source = s3Source("survival-source"); + source.getSpec().getS3().setCredentialsSecretRef(new net.onelitefeather.apus.operator.api.Ref()); + + Job job = IngestJobBuilder.build(ingest("i1", "v1"), source, OperatorConfig.defaults()); + + assertNull( + envOf(job).get("APUS_SOURCE_S3_ACCESS_KEY"), + "no secret configured -- the connector falls back to the AWS default credentials chain"); + } + + @Test + void suppliesPterodactylSpecificVariablesDerivedFromTheBukkitWorldSplitConvention() { + Job job = IngestJobBuilder.build( + ingest("i1", "backup-uuid"), pterodactylSource("survival-source"), OperatorConfig.defaults()); + + Map env = envOf(job); + + assertEquals("pterodactyl", env.get("APUS_SOURCE_TYPE").getValue()); + assertEquals("https://panel.example.com", env.get("APUS_PTERODACTYL_PANEL_URL").getValue()); + assertEquals("abc123", env.get("APUS_PTERODACTYL_SERVER_ID").getValue()); + assertEquals("panel-creds", env.get("APUS_PTERODACTYL_API_KEY").getValueFrom().getSecretKeyRef().getName()); + assertEquals("world,world_nether,world_the_end", env.get("APUS_PTERODACTYL_WORLD_PATHS").getValue()); + assertNull(env.get("APUS_SOURCE_S3_BUCKET"), "an S3-only variable must not leak into a pterodactyl job"); + } + + @Test + void layoutDefaultsToAutoWhenNoSelectorMatchesTheWorldName() { + WorldSource source = s3Source("survival-source"); + source.getSpec().getWorlds().clear(); + + Job job = IngestJobBuilder.build(ingest("i1", "v1"), source, OperatorConfig.defaults()); + + assertEquals("auto", envOf(job).get("APUS_LAYOUT").getValue()); + } + + @Test + void placesTheContainerImageFromTheOperatorConfig() { + OperatorConfig config = new OperatorConfig( + "rook-ceph-fr01", + "feather-s3", + "ceph-bucket-fr01", + "apus/runner:dev", + "apus/ingest:1.2.3", + "apus-bundles", + "http://rgw.example.svc:80", + "us-east-1", + "apus-bundle-credentials"); + + Job job = IngestJobBuilder.build(ingest("i1", "v1"), s3Source("survival-source"), config); + + assertEquals( + "apus/ingest:1.2.3", + job.getSpec().getTemplate().getSpec().getContainers().get(0).getImage()); + } + + @Test + void ownsTheJobViaAnOwnerReferenceToTheIngestNameAndUid() { + WorldIngest ingest = ingest("survival-source-world-v1", "v1"); + + Job job = IngestJobBuilder.build(ingest, s3Source("survival-source"), OperatorConfig.defaults()); + + OwnerReference owner = + job.getMetadata().getOwnerReferences().get(0); + assertEquals("WorldIngest", owner.getKind()); + assertEquals(ingest.getMetadata().getName(), owner.getName()); + assertEquals(ingest.getMetadata().getUid(), owner.getUid()); + } + + @Test + void labelsRecordTheOwningSourceForCrossResourceQueries() { + WorldSource source = s3Source("survival-source"); + Job job = IngestJobBuilder.build(ingest("i1", "v1"), source, OperatorConfig.defaults()); + + Map labels = job.getMetadata().getLabels(); + assertEquals(Labels.MANAGED_BY_VALUE, labels.get(Labels.MANAGED_BY)); + assertEquals("survival-source", labels.get(Labels.SOURCE)); + assertEquals(source.getMetadata().getUid(), labels.get(Labels.SOURCE_UID)); + } +} diff --git a/operator/src/test/java/net/onelitefeather/apus/operator/ingest/IngestLogProgressTest.java b/operator/src/test/java/net/onelitefeather/apus/operator/ingest/IngestLogProgressTest.java new file mode 100644 index 0000000..20e6fd9 --- /dev/null +++ b/operator/src/test/java/net/onelitefeather/apus/operator/ingest/IngestLogProgressTest.java @@ -0,0 +1,89 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.ingest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.junit.jupiter.api.Test; + +class IngestLogProgressTest { + + @Test + void extractsTheLastPhaseLine() { + String log = """ + [apus-ingest] phase=Pending source=s3 world=world bundle=t/w/v1 + [apus-ingest] phase=Extracting + [apus-ingest] phase=Transforming + """; + + IngestLogProgress progress = IngestLogProgress.parse(log); + + assertEquals("Transforming", progress.phase()); + } + + @Test + void extractsTheLastProgressLine() { + String log = """ + [apus-ingest] phase=Loading + [apus-ingest] progress: 10.0% (100/1000 bytes) + [apus-ingest] progress: 55.5% (555/1000 bytes) + """; + + IngestLogProgress progress = IngestLogProgress.parse(log); + + assertEquals(55.5, progress.percent()); + assertEquals(555L, progress.bytesDone()); + assertEquals(1000L, progress.bytesTotal()); + } + + @Test + void extractsDimensionsFromTheDetectedLayoutLine() { + String log = "[apus-ingest] detected layout kind=bukkit dimensions=[overworld, the_nether, the_end]"; + + IngestLogProgress progress = IngestLogProgress.parse(log); + + assertEquals(List.of("overworld", "the_nether", "the_end"), progress.dimensions()); + } + + @Test + void emptyOrMissingLinesYieldNullFieldsRatherThanGuessedValues() { + IngestLogProgress progress = IngestLogProgress.parse(""); + + assertNull(progress.phase()); + assertNull(progress.percent()); + assertTrue(progress.dimensions().isEmpty()); + } + + @Test + void nullLogYieldsAllNullFieldsWithoutThrowing() { + IngestLogProgress progress = IngestLogProgress.parse(null); + + assertNull(progress.phase()); + assertTrue(progress.dimensions().isEmpty()); + } + + @Test + void recognisesTheFinalSucceededPhaseAlongsideItsBundlePathSuffix() { + String log = "[apus-ingest] phase=Succeeded bundlePath=acme/survival/v1"; + + assertEquals("Succeeded", IngestLogProgress.parse(log).phase()); + } +} diff --git a/operator/src/test/java/net/onelitefeather/apus/operator/ingest/WorldIngestReconcilerTest.java b/operator/src/test/java/net/onelitefeather/apus/operator/ingest/WorldIngestReconcilerTest.java new file mode 100644 index 0000000..e90d9e9 --- /dev/null +++ b/operator/src/test/java/net/onelitefeather/apus/operator/ingest/WorldIngestReconcilerTest.java @@ -0,0 +1,698 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.ingest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.api.model.batch.v1.Job; +import io.fabric8.kubernetes.api.model.batch.v1.JobBuilder; +import io.fabric8.kubernetes.api.model.batch.v1.JobConditionBuilder; +import io.fabric8.kubernetes.api.model.batch.v1.JobStatus; +import io.fabric8.kubernetes.api.model.batch.v1.JobStatusBuilder; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import net.onelitefeather.apus.operator.OperatorConfig; +import net.onelitefeather.apus.operator.api.BlueMapRender; +import net.onelitefeather.apus.operator.api.Conditions; +import net.onelitefeather.apus.operator.api.Labels; +import net.onelitefeather.apus.operator.api.WorldIngest; +import net.onelitefeather.apus.operator.api.WorldSource; +import org.junit.jupiter.api.Test; + +@EnableKubernetesMockClient(crud = true) +class WorldIngestReconcilerTest { + + KubernetesClient client; + + private WorldSource source(String name) { + WorldSource source = new WorldSource(); + source.setMetadata(new ObjectMetaBuilder() + .withName(name) + .withNamespace("bluemap-friends") + .withUid(UUID.randomUUID().toString()) + .build()); + source.getSpec().setType("s3"); + source.getSpec().getS3().setBucket("backups"); + return source; + } + + /** + * Creates {@code name} via the (mock) API server and returns the server-fetched copy -- + * needed because the fabric8 CRUD mock server assigns its own UID on create, ignoring + * whatever the client supplied (exactly like a real API server would). {@link #ingest(String, + * WorldSource)} needs the *actual* UID to stamp owner labels {@code WorldIngestReconciler}'s + * ownership check will accept, so every test that creates a source goes through here rather + * than the raw {@link #source(String)} builder plus a bare {@code .create()}. + */ + private WorldSource createSource(String name) { + WorldSource source = source(name); + client.resources(WorldSource.class).inNamespace("bluemap-friends").resource(source).create(); + return client.resources(WorldSource.class).inNamespace("bluemap-friends").withName(name).get(); + } + + /** An ingest whose {@code sourceRef} points at a source that was never created. */ + private WorldIngest ingest(String name) { + WorldIngest ingest = new WorldIngest(); + ingest.setMetadata(new ObjectMetaBuilder() + .withName(name) + .withNamespace("bluemap-friends") + .withUid(UUID.randomUUID().toString()) + .build()); + ingest.getSpec().getSourceRef().setName("survival-source"); + ingest.getSpec().setSourceVersion("v1.zip"); + ingest.getSpec().setWorldName("world"); + return ingest; + } + + /** + * An ingest legitimately triggered for {@code source} -- carries the exact owner labels + * {@code WorldSourceReconciler} stamps (name and UID), which {@code WorldIngestReconciler}'s + * ownership check requires before it will act on the source at all. + */ + private WorldIngest ingest(String name, WorldSource source) { + WorldIngest ingest = new WorldIngest(); + ingest.setMetadata(new ObjectMetaBuilder() + .withName(name) + .withNamespace("bluemap-friends") + .withUid(UUID.randomUUID().toString()) + .addToLabels(Labels.SOURCE, source.getMetadata().getName()) + .addToLabels(Labels.SOURCE_UID, source.getMetadata().getUid()) + .build()); + ingest.getSpec().getSourceRef().setName(source.getMetadata().getName()); + ingest.getSpec().setSourceVersion("v1.zip"); + ingest.getSpec().setWorldName("world"); + return ingest; + } + + private String readyReason(WorldIngest ingest) { + return ingest.getStatus().getConditions().stream() + .filter(c -> Conditions.READY.equals(c.getType())) + .findFirst() + .orElseThrow() + .getReason(); + } + + /** A fake {@link BundleStore} the tests can inspect/pre-seed without touching real S3. */ + private static final class FakeBundleStore implements BundleStore { + final Map versions = new HashMap<>(); + final List deleted = new ArrayList<>(); + + void seed(String version, Instant lastModified) { + versions.put(version, lastModified); + } + + @Override + public List listVersions(String tenant, String sourceName, String worldId, String bundleBucket) { + List result = new ArrayList<>(); + versions.forEach((version, time) -> result.add(new BundleVersion(version, time))); + return result; + } + + @Override + public void deleteVersion(String tenant, String sourceName, String worldId, String version, String bundleBucket) { + versions.remove(version); + deleted.add(version); + } + } + + /** A Job status with a {@code Failed} condition -- what the Job controller sets once {@code backoffLimit} is exhausted. */ + private static JobStatus failedJobStatus() { + return new JobStatusBuilder() + .withConditions(new JobConditionBuilder() + .withType("Failed") + .withStatus("True") + .build()) + .build(); + } + + private WorldIngestReconciler reconciler(FakeBundleStore store) { + return new WorldIngestReconciler( + client, + OperatorConfig.defaults(), + src -> client.resources(WorldSource.class) + .inNamespace(src.getMetadata().getNamespace()) + .resource(src) + .updateStatus(), + pod -> Optional.empty(), + destination -> store); + } + + private WorldIngestReconciler reconcilerWithLog(FakeBundleStore store, String log) { + return new WorldIngestReconciler( + client, + OperatorConfig.defaults(), + src -> client.resources(WorldSource.class) + .inNamespace(src.getMetadata().getNamespace()) + .resource(src) + .updateStatus(), + pod -> Optional.of(log), + destination -> store); + } + + private void markJobSucceeded(String jobName) { + Job job = client.batch().v1().jobs().inNamespace("bluemap-friends").withName(jobName).get(); + job.setStatus(new JobStatusBuilder().withSucceeded(1).build()); + client.batch() + .v1() + .jobs() + .inNamespace("bluemap-friends") + .resource(job) + .createOr(io.fabric8.kubernetes.client.dsl.NonDeletingOperation::update); + } + + // --- Job submission ------------------------------------------------------------------- + + @Test + void submitsAnIngestJobAndClaimsTheSourceLock() { + WorldSource source = createSource("survival-source"); + WorldIngest ingest = ingest("ingest-1", source); + + reconciler(new FakeBundleStore()).reconcile(ingest, null); + + Job job = client.batch().v1().jobs().inNamespace("bluemap-friends").withName("ingest-1").get(); + assertNotNullJob(job); + assertEquals("Extracting", ingest.getStatus().getPhase()); + + WorldSource updated = + client.resources(WorldSource.class).inNamespace("bluemap-friends").withName("survival-source").get(); + assertEquals("ingest-1", updated.getStatus().getActiveIngest().getName()); + } + + private static void assertNotNullJob(Job job) { + if (job == null) { + throw new AssertionError("expected the ingest job to have been created"); + } + } + + @Test + void refusesToStartASecondIngestForTheSameSource() { + WorldSource source = createSource("survival-source"); + + WorldIngest first = ingest("ingest-1", source); + reconciler(new FakeBundleStore()).reconcile(first, null); + + WorldIngest second = ingest("ingest-2", source); + reconciler(new FakeBundleStore()).reconcile(second, null); + + assertEquals("Pending", second.getStatus().getPhase()); + assertNull(client.batch().v1().jobs().inNamespace("bluemap-friends").withName("ingest-2").get()); + } + + @Test + void aSecondIngestProceedsOnceTheFirstIsTerminal() { + WorldSource source = createSource("survival-source"); + + WorldIngest first = ingest("ingest-1", source); + reconciler(new FakeBundleStore()).reconcile(first, null); + // Simulate the first ingest's job failing terminally -- anotherActiveIngestJobExists() + // checks the Job's own status, exactly like BlueMapRenderReconciler's twin does for + // renders, so the underlying Job (not just the WorldIngest CR) must reflect failure. + Job firstJob = client.batch().v1().jobs().inNamespace("bluemap-friends").withName("ingest-1").get(); + firstJob.setStatus(failedJobStatus()); + client.batch() + .v1() + .jobs() + .inNamespace("bluemap-friends") + .resource(firstJob) + .createOr(io.fabric8.kubernetes.client.dsl.NonDeletingOperation::update); + first.getStatus().setPhase("Failed"); + client.resources(WorldIngest.class).inNamespace("bluemap-friends").resource(first).create(); + + WorldIngest second = ingest("ingest-2", source); + reconciler(new FakeBundleStore()).reconcile(second, null); + + assertNotNullJob(client.batch().v1().jobs().inNamespace("bluemap-friends").withName("ingest-2").get()); + } + + @Test + void reconcilingAnAlreadyOwnedJobDoesNotRecreateIt() { + WorldSource source = createSource("survival-source"); + WorldIngest ingest = ingest("ingest-1", source); + + WorldIngestReconciler reconciler = reconciler(new FakeBundleStore()); + reconciler.reconcile(ingest, null); + String firstResourceVersion = client.batch() + .v1() + .jobs() + .inNamespace("bluemap-friends") + .withName("ingest-1") + .get() + .getMetadata() + .getResourceVersion(); + + reconciler.reconcile(ingest, null); + String secondResourceVersion = client.batch() + .v1() + .jobs() + .inNamespace("bluemap-friends") + .withName("ingest-1") + .get() + .getMetadata() + .getResourceVersion(); + + assertEquals(firstResourceVersion, secondResourceVersion, "reconciling twice must be idempotent"); + } + + @Test + void sourceNotFoundIsReportedInsteadOfThrowing() { + WorldIngest ingest = ingest("ingest-1"); // sourceRef points at a source never created + + reconciler(new FakeBundleStore()).reconcile(ingest, null); + + assertEquals(WorldIngestReconciler.SOURCE_NOT_FOUND_REASON, readyReason(ingest)); + } + + @Test + void aJobNotOwnedByThisIngestIsReportedAsAConflict() { + WorldSource source = createSource("survival-source"); + + Job foreignJob = new JobBuilder() + .withNewMetadata() + .withName("ingest-1") + .withNamespace("bluemap-friends") + .endMetadata() + .withNewSpec() + .withNewTemplate() + .withNewSpec() + .withRestartPolicy("Never") + .endSpec() + .endTemplate() + .endSpec() + .build(); // no owner reference at all + client.batch().v1().jobs().inNamespace("bluemap-friends").resource(foreignJob).create(); + + WorldIngest ingest = ingest("ingest-1", source); + reconciler(new FakeBundleStore()).reconcile(ingest, null); + + assertEquals(WorldIngestReconciler.RESOURCE_CONFLICT_REASON, readyReason(ingest)); + } + + /** + * S3: {@code WorldIngestReconciler} must not trust {@code spec.sourceRef.name} alone -- a + * hand-written or stale {@link WorldIngest} (e.g. its original source was deleted and a + * different source created under the same name, giving it a different UID) must not be able + * to read/overwrite that source's status or drive retention against its bundles. Only an + * ingest carrying the exact owner labels {@code WorldSourceReconciler} stamps (name AND UID) + * may proceed. + */ + @Test + void anIngestWithoutTheSourceOwnerLabelsIsReportedAsAConflictAndNeverTouchesTheSource() { + createSource("survival-source"); + + // Same sourceRef.name as a legitimately-triggered ingest, but no owner labels at all -- + // exactly what a hand-written WorldIngest (or one whose original source was deleted and + // recreated under the same name, giving it a different UID) would look like. + WorldIngest rogue = new WorldIngest(); + rogue.setMetadata(new ObjectMetaBuilder() + .withName("rogue-ingest") + .withNamespace("bluemap-friends") + .withUid(UUID.randomUUID().toString()) + .build()); + rogue.getSpec().getSourceRef().setName("survival-source"); + rogue.getSpec().setSourceVersion("v1.zip"); + rogue.getSpec().setWorldName("world"); + + reconciler(new FakeBundleStore()).reconcile(rogue, null); + + assertEquals(WorldIngestReconciler.RESOURCE_CONFLICT_REASON, readyReason(rogue)); + assertNull( + client.batch().v1().jobs().inNamespace("bluemap-friends").withName("rogue-ingest").get(), + "no job may be submitted for an ingest that fails the source ownership check"); + + WorldSource unchanged = + client.resources(WorldSource.class).inNamespace("bluemap-friends").withName("survival-source").get(); + assertNull( + unchanged.getStatus().getActiveIngest().getName(), + "the source's status must be untouched by an ingest that does not own it"); + } + + @Test + void reconcilingATerminalIngestIsANoOp() { + WorldIngest ingest = ingest("ingest-1"); + ingest.getStatus().setPhase("Succeeded"); + + var control = reconciler(new FakeBundleStore()).reconcile(ingest, null); + + assertTrue(control.isNoUpdate()); + } + + // --- Job progress / completion -------------------------------------------------------- + + @Test + void aFailedJobMarksTheIngestFailed() { + WorldSource source = createSource("survival-source"); + WorldIngest ingest = ingest("ingest-1", source); + WorldIngestReconciler reconciler = reconciler(new FakeBundleStore()); + reconciler.reconcile(ingest, null); + + Job job = client.batch().v1().jobs().inNamespace("bluemap-friends").withName("ingest-1").get(); + job.setStatus(failedJobStatus()); + client.batch().v1().jobs().inNamespace("bluemap-friends").resource(job).createOr( + io.fabric8.kubernetes.client.dsl.NonDeletingOperation::update); + + reconciler.reconcile(ingest, null); + + assertEquals("Failed", ingest.getStatus().getPhase()); + assertEquals(WorldIngestReconciler.JOB_FAILED_REASON, readyReason(ingest)); + } + + /** + * B2: {@code backoffLimit} exists precisely so a transient pod failure gets retried -- a + * single failed *pod attempt* ({@code status.failed > 0}, no {@code Failed} condition yet) + * must not end the ingest terminally while the Job controller is still going to retry it. If + * it did, the CR would go terminal here, the Job would keep running underneath it, and a + * later attempt could still write a complete bundle that then has nowhere to be registered -- + * a terminal ingest is never reconciled again. + */ + @Test + void aSingleFailedPodAttemptDoesNotEndTheIngestWhileTheJobStillHasRetriesLeft() { + WorldSource source = createSource("survival-source"); + WorldIngest ingest = ingest("ingest-1", source); + WorldIngestReconciler reconciler = reconciler(new FakeBundleStore()); + reconciler.reconcile(ingest, null); + + // One failed pod attempt recorded, but the Job controller has not (yet) exhausted + // backoffLimit -- no Failed condition set. This is exactly the state a Job is in + // between a transient pod crash and its next retry attempt. + Job job = client.batch().v1().jobs().inNamespace("bluemap-friends").withName("ingest-1").get(); + job.setStatus(new JobStatusBuilder().withFailed(1).build()); + client.batch().v1().jobs().inNamespace("bluemap-friends").resource(job).createOr( + io.fabric8.kubernetes.client.dsl.NonDeletingOperation::update); + + reconciler.reconcile(ingest, null); + + assertFalse("Failed".equals(ingest.getStatus().getPhase()), "a single failed attempt must not be terminal"); + assertEquals("Extracting", ingest.getStatus().getPhase(), "the ingest must still be treated as running"); + } + + @Test + void aSucceededJobMarksTheIngestSucceededAndFillsTheBundleRef() { + WorldSource source = createSource("survival-source"); + WorldIngest ingest = ingest("ingest-1", source); + WorldIngestReconciler reconciler = reconciler(new FakeBundleStore()); + reconciler.reconcile(ingest, null); + + Job job = client.batch().v1().jobs().inNamespace("bluemap-friends").withName("ingest-1").get(); + job.setStatus(new JobStatusBuilder().withSucceeded(1).build()); + client.batch().v1().jobs().inNamespace("bluemap-friends").resource(job).createOr( + io.fabric8.kubernetes.client.dsl.NonDeletingOperation::update); + + reconciler.reconcile(ingest, null); + + assertEquals("Succeeded", ingest.getStatus().getPhase()); + assertEquals("friends/survival-source/world/ingest-1", ingest.getStatus().getBundle().getPath()); + assertEquals("ingest-1", ingest.getStatus().getBundle().getVersion()); + + WorldSource updated = + client.resources(WorldSource.class).inNamespace("bluemap-friends").withName("survival-source").get(); + assertEquals("friends/survival-source/world/ingest-1", updated.getStatus().getLatestBundle().getPath()); + } + + @Test + void aSucceededJobFillsDimensionsParsedFromThePodLog() { + WorldSource source = createSource("survival-source"); + WorldIngest ingest = ingest("ingest-1", source); + String log = "[apus-ingest] detected layout kind=bukkit dimensions=[overworld, the_nether]"; + WorldIngestReconciler reconciler = reconcilerWithLog(new FakeBundleStore(), log); + reconciler.reconcile(ingest, null); + + Job job = client.batch().v1().jobs().inNamespace("bluemap-friends").withName("ingest-1").get(); + job.setStatus(new JobStatusBuilder().withSucceeded(1).build()); + client.batch().v1().jobs().inNamespace("bluemap-friends").resource(job).createOr( + io.fabric8.kubernetes.client.dsl.NonDeletingOperation::update); + // findPod() locates the pod by the "job-name" label the Job controller always stamps. + io.fabric8.kubernetes.api.model.Pod pod = new io.fabric8.kubernetes.api.model.PodBuilder() + .withNewMetadata() + .withName("ingest-1-abcde") + .withNamespace("bluemap-friends") + .addToLabels("job-name", "ingest-1") + .endMetadata() + .build(); + client.pods().inNamespace("bluemap-friends").resource(pod).create(); + + reconciler.reconcile(ingest, null); + + assertEquals("Succeeded", ingest.getStatus().getPhase()); + assertEquals(List.of("overworld", "the_nether"), ingest.getStatus().getBundle().getDimensions()); + } + + /** + * C1: {@code IngestMain} logs {@code phase=Succeeded} strictly before the process exits -- + * i.e. strictly before the Kubernetes Job controller can have observed the pod's exit and set + * {@code status.succeeded}. A reconcile landing in exactly that window must not adopt + * {@code Succeeded} out of the log: terminality belongs only to the Job's own status. Without + * the fix, this reconcile would set the ingest's phase terminal here, {@link + * WorldIngestReconciler#reconcile} would then treat every future reconcile of this ingest as + * a no-op (see the class Javadoc), and {@code onJobSucceeded} -- which fills {@code + * status.bundle} and {@code WorldSource.status.latestBundle} -- would never run, even once the + * Job controller genuinely reports success afterwards. + */ + @Test + void logReportingSucceededBeforeTheJobStatusDoesNotEndTheIngestPrematurely() { + WorldSource source = createSource("survival-source"); + WorldIngest ingest = ingest("ingest-1", source); + // The exact race: IngestMain's last log line already says Succeeded... + String log = "[apus-ingest] phase=Succeeded bundlePath=friends/survival-source/world/ingest-1"; + WorldIngestReconciler reconciler = reconcilerWithLog(new FakeBundleStore(), log); + reconciler.reconcile(ingest, null); // submits the job + + // ...but the Job controller has not observed the pod's exit yet -- status is still empty. + io.fabric8.kubernetes.api.model.Pod pod = new io.fabric8.kubernetes.api.model.PodBuilder() + .withNewMetadata() + .withName("ingest-1-abcde") + .withNamespace("bluemap-friends") + .addToLabels("job-name", "ingest-1") + .endMetadata() + .build(); + client.pods().inNamespace("bluemap-friends").resource(pod).create(); + + reconciler.reconcile(ingest, null); + + assertFalse( + "Succeeded".equals(ingest.getStatus().getPhase()), + "the log alone must never make the ingest terminal -- only the Job's own status may"); + assertNull( + ingest.getStatus().getBundle().getPath(), "the bundle must not be registered before the job actually succeeded"); + + // Now the Job controller catches up for real. + Job job = client.batch().v1().jobs().inNamespace("bluemap-friends").withName("ingest-1").get(); + job.setStatus(new JobStatusBuilder().withSucceeded(1).build()); + client.batch().v1().jobs().inNamespace("bluemap-friends").resource(job).createOr( + io.fabric8.kubernetes.client.dsl.NonDeletingOperation::update); + + reconciler.reconcile(ingest, null); + + assertEquals("Succeeded", ingest.getStatus().getPhase()); + assertEquals( + "friends/survival-source/world/ingest-1", + ingest.getStatus().getBundle().getPath(), + "the bundle must be registered once the job genuinely succeeded"); + WorldSource updated = + client.resources(WorldSource.class).inNamespace("bluemap-friends").withName("survival-source").get(); + assertEquals( + "friends/survival-source/world/ingest-1", + updated.getStatus().getLatestBundle().getPath(), + "WorldSource.status.latestBundle must be filled once the job genuinely succeeded"); + } + + // --- Retention -------------------------------------------------------------------------- + + @Test + void retentionDeletesOlderVersionsBeyondKeepVersions() { + WorldSource source = source("survival-source"); + source.getSpec().getRetention().setKeepVersions(2); + client.resources(WorldSource.class).inNamespace("bluemap-friends").resource(source).create(); + source = client.resources(WorldSource.class).inNamespace("bluemap-friends").withName("survival-source").get(); + + FakeBundleStore store = new FakeBundleStore(); + store.seed("old-1", Instant.parse("2026-08-01T00:00:00Z")); + store.seed("old-2", Instant.parse("2026-08-02T00:00:00Z")); + store.seed("old-3", Instant.parse("2026-08-03T00:00:00Z")); + + WorldIngest ingest = ingest("ingest-1", source); // this run's own version becomes the newest + WorldIngestReconciler reconciler = reconciler(store); + reconciler.reconcile(ingest, null); + + Job job = client.batch().v1().jobs().inNamespace("bluemap-friends").withName("ingest-1").get(); + job.setStatus(new JobStatusBuilder().withSucceeded(1).build()); + client.batch().v1().jobs().inNamespace("bluemap-friends").resource(job).createOr( + io.fabric8.kubernetes.client.dsl.NonDeletingOperation::update); + + reconciler.reconcile(ingest, null); + + // keepVersions=2 plus the just-written "ingest-1" version (newest) = 3 kept; the two + // oldest of the three seeded versions must be gone. + assertTrue(store.deleted.contains("old-1")); + assertFalse(store.versions.containsKey("old-1")); + assertTrue(store.versions.containsKey("old-3"), "the newest of the pre-existing versions must survive"); + } + + @Test + void retentionNeverDeletesAVersionStillReferencedByABlueMapRender() { + WorldSource source = source("survival-source"); + source.getSpec().getRetention().setKeepVersions(0); // would prune everything without the guard + client.resources(WorldSource.class).inNamespace("bluemap-friends").resource(source).create(); + source = client.resources(WorldSource.class).inNamespace("bluemap-friends").withName("survival-source").get(); + + FakeBundleStore store = new FakeBundleStore(); + store.seed("referenced-version", Instant.parse("2026-08-01T00:00:00Z")); + + BlueMapRender render = new BlueMapRender(); + render.setMetadata( + new ObjectMetaBuilder().withName("r1").withNamespace("bluemap-friends").build()); + render.getSpec() + .setBundleUrl( + "s3://bundles/friends/survival-source/world/referenced-version/dimensions/overworld/region"); + client.resources(BlueMapRender.class).inNamespace("bluemap-friends").resource(render).create(); + + WorldIngest ingest = ingest("ingest-1", source); + WorldIngestReconciler reconciler = reconciler(store); + reconciler.reconcile(ingest, null); + + Job job = client.batch().v1().jobs().inNamespace("bluemap-friends").withName("ingest-1").get(); + job.setStatus(new JobStatusBuilder().withSucceeded(1).build()); + client.batch().v1().jobs().inNamespace("bluemap-friends").resource(job).createOr( + io.fabric8.kubernetes.client.dsl.NonDeletingOperation::update); + + reconciler.reconcile(ingest, null); + + assertTrue( + store.versions.containsKey("referenced-version"), + "a version a BlueMapRender still references must never be deleted, even beyond keepVersions"); + assertFalse(store.deleted.contains("referenced-version")); + } + + @Test + void retentionKeepsTheJustWrittenVersionEvenWithKeepVersionsZero() { + WorldSource source = source("survival-source"); + source.getSpec().getRetention().setKeepVersions(0); + client.resources(WorldSource.class).inNamespace("bluemap-friends").resource(source).create(); + source = client.resources(WorldSource.class).inNamespace("bluemap-friends").withName("survival-source").get(); + + WorldIngest ingest = ingest("ingest-1", source); + WorldIngestReconciler reconciler = reconciler(new FakeBundleStore()); + reconciler.reconcile(ingest, null); + + Job job = client.batch().v1().jobs().inNamespace("bluemap-friends").withName("ingest-1").get(); + job.setStatus(new JobStatusBuilder().withSucceeded(1).build()); + client.batch().v1().jobs().inNamespace("bluemap-friends").resource(job).createOr( + io.fabric8.kubernetes.client.dsl.NonDeletingOperation::update); + + // Must not throw even though the FakeBundleStore never had "ingest-1" seeded into it + // (a real S3 listVersions() call would see it because the job actually wrote it). + reconciler.reconcile(ingest, null); + + assertEquals("Succeeded", ingest.getStatus().getPhase()); + } + + /** + * C2 (part 1): the bundle path must be scoped by the owning source's name, not just {@code + * tenant}/{@code worldId} -- two different {@link WorldSource}s in the same namespace both + * ingesting a world literally named {@code "world"} (the Minecraft default) must never + * resolve to the same bundle path. + */ + @Test + void twoSourcesWithTheSameWorldNameNeverCollideOnTheSameBundlePath() { + WorldSource sourceA = createSource("source-a"); + WorldSource sourceB = createSource("source-b"); + + WorldIngest ingestA = ingest("ingest-a", sourceA); + WorldIngest ingestB = ingest("ingest-b", sourceB); + WorldIngestReconciler reconciler = reconciler(new FakeBundleStore()); + reconciler.reconcile(ingestA, null); + reconciler.reconcile(ingestB, null); + + markJobSucceeded("ingest-a"); + markJobSucceeded("ingest-b"); + reconciler.reconcile(ingestA, null); + reconciler.reconcile(ingestB, null); + + assertEquals("friends/source-a/world/ingest-a", ingestA.getStatus().getBundle().getPath()); + assertEquals("friends/source-b/world/ingest-b", ingestB.getStatus().getBundle().getPath()); + assertFalse( + ingestA.getStatus().getBundle().getPath().equals(ingestB.getStatus().getBundle().getPath()), + "two different sources ingesting the same world name must never share a bundle path"); + + WorldSource updatedA = + client.resources(WorldSource.class).inNamespace("bluemap-friends").withName("source-a").get(); + WorldSource updatedB = + client.resources(WorldSource.class).inNamespace("bluemap-friends").withName("source-b").get(); + assertEquals("friends/source-a/world/ingest-a", updatedA.getStatus().getLatestBundle().getPath()); + assertEquals("friends/source-b/world/ingest-b", updatedB.getStatus().getLatestBundle().getPath()); + } + + /** + * C2 (part 2): retention must never delete a bundle version recorded as any {@link + * WorldSource}'s {@code status.latestBundle} in the namespace, not just the source the + * current retention pass belongs to -- a safety net independent of (and in addition to) the + * source-scoped path prefix from part 1; see {@code applyRetention}'s Javadoc. + */ + @Test + void retentionNeverDeletesAVersionRecordedAsAnotherSourcesLatestBundle() { + WorldSource sourceA = source("source-a"); + sourceA.getSpec().getRetention().setKeepVersions(0); // would prune everything without the guard + client.resources(WorldSource.class).inNamespace("bluemap-friends").resource(sourceA).create(); + sourceA = client.resources(WorldSource.class).inNamespace("bluemap-friends").withName("source-a").get(); + + WorldSource sourceB = source("source-b"); + sourceB.getStatus().getLatestBundle().setPath("friends/source-a/world/shared-version"); + client.resources(WorldSource.class).inNamespace("bluemap-friends").resource(sourceB).create(); + + FakeBundleStore store = new FakeBundleStore(); + store.seed("shared-version", Instant.parse("2026-08-01T00:00:00Z")); + + WorldIngest ingest = ingest("ingest-1", sourceA); + WorldIngestReconciler reconciler = reconciler(store); + reconciler.reconcile(ingest, null); + markJobSucceeded("ingest-1"); + reconciler.reconcile(ingest, null); + + assertTrue( + store.versions.containsKey("shared-version"), + "a version recorded as another source's latestBundle must never be deleted"); + assertFalse(store.deleted.contains("shared-version")); + } + + // --- referencesBundle boundary safety --------------------------------------------------- + + @Test + void referencesBundleDoesNotMatchAVersionThatIsOnlyAPrefixOfAnother() { + assertFalse(WorldIngestReconciler.referencesBundle("s3://bundles/t/w/v10/dimensions/overworld", "t/w/v1")); + } + + @Test + void referencesBundleMatchesAnExactPathSegment() { + assertTrue(WorldIngestReconciler.referencesBundle("s3://bundles/t/w/v1/dimensions/overworld", "t/w/v1")); + } + + @Test + void referencesBundleMatchesWhenTheVersionIsTheEntireTrailingPath() { + assertTrue(WorldIngestReconciler.referencesBundle("s3://bundles/t/w/v1", "t/w/v1")); + } +} diff --git a/operator/src/test/java/net/onelitefeather/apus/operator/ingest/WorldSourceReconcilerTest.java b/operator/src/test/java/net/onelitefeather/apus/operator/ingest/WorldSourceReconcilerTest.java new file mode 100644 index 0000000..81eb113 --- /dev/null +++ b/operator/src/test/java/net/onelitefeather/apus/operator/ingest/WorldSourceReconcilerTest.java @@ -0,0 +1,312 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.operator.ingest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import net.onelitefeather.apus.ingest.connector.SourceVersion; +import net.onelitefeather.apus.ingest.connector.WorldSourceConnector; +import net.onelitefeather.apus.operator.api.Conditions; +import net.onelitefeather.apus.operator.api.WorldIngest; +import net.onelitefeather.apus.operator.api.WorldSource; +import org.junit.jupiter.api.Test; + +@EnableKubernetesMockClient(crud = true) +class WorldSourceReconcilerTest { + + KubernetesClient client; + + private static final Clock FIXED_CLOCK = Clock.fixed(Instant.parse("2026-08-09T12:00:00Z"), ZoneOffset.UTC); + + private WorldSource source(String name) { + WorldSource source = new WorldSource(); + source.setMetadata(new ObjectMetaBuilder() + .withName(name) + .withNamespace("bluemap-friends") + .withUid(UUID.randomUUID().toString()) + .build()); + source.getSpec().setType("s3"); + source.getSpec().setPoll("0 * * * *"); // hourly + source.getSpec().getS3().setBucket("backups"); + WorldSource.WorldSelector selector = new WorldSource.WorldSelector(); + selector.setName("world"); + source.getSpec().getWorlds().add(selector); + return source; + } + + private String readyReason(WorldSource source) { + return source.getStatus().getConditions().stream() + .filter(c -> Conditions.READY.equals(c.getType())) + .findFirst() + .orElseThrow() + .getReason(); + } + + private WorldSourceReconciler reconciler(WorldSourceConnector connector) { + return new WorldSourceReconciler(client, type -> connector, FIXED_CLOCK); + } + + private WorldSourceConnector fixedVersions(List versions) { + return new WorldSourceConnector() { + @Override + public String type() { + return "s3"; + } + + @Override + public List discover(Map config) { + return versions; + } + + @Override + public void fetch(Map config, SourceVersion version, java.nio.file.Path workDir) { + throw new UnsupportedOperationException("not used by WorldSourceReconciler"); + } + }; + } + + @Test + void manualOnlySourceIsNeverPolled() { + WorldSource source = source("s1"); + source.getSpec().setPoll(null); + + WorldSourceReconciler reconciler = reconciler(fixedVersions(List.of())); + reconciler.reconcile(source, null); + + assertEquals(WorldSourceReconciler.MANUAL_ONLY_REASON, readyReason(source)); + } + + @Test + void pushTypeSourceIsNeverPolledEvenIfPollIsSet() { + WorldSource source = source("s1"); + source.getSpec().setType("upload"); + source.getSpec().setPoll("0 * * * *"); + + WorldSourceReconciler reconciler = reconciler(fixedVersions(List.of())); + reconciler.reconcile(source, null); + + assertEquals(WorldSourceReconciler.MANUAL_ONLY_REASON, readyReason(source)); + } + + @Test + void invalidCronExpressionIsReportedRatherThanGuessed() { + WorldSource source = source("s1"); + source.getSpec().setPoll("not a cron"); + + WorldSourceReconciler reconciler = reconciler(fixedVersions(List.of())); + reconciler.reconcile(source, null); + + assertEquals(WorldSourceReconciler.INVALID_POLL_REASON, readyReason(source)); + } + + @Test + void notYetDueDoesNotCallDiscoverAtAll() { + WorldSource source = source("s1"); + // Already polled once this hour (11:00); the hourly cron's next fire is 12:00, and + // "now" below is still well before that. + source.getStatus().setLastPollTime(Instant.parse("2026-08-09T11:00:00Z").toString()); + + WorldSourceConnector connector = new WorldSourceConnector() { + @Override + public String type() { + return "s3"; + } + + @Override + public List discover(Map config) { + throw new AssertionError("discover() must not be called before the poll is due"); + } + + @Override + public void fetch(Map config, SourceVersion version, java.nio.file.Path workDir) {} + }; + + Clock notYetDueClock = Clock.fixed(Instant.parse("2026-08-09T11:50:00Z"), ZoneOffset.UTC); + var control = new WorldSourceReconciler(client, type -> connector, notYetDueClock).reconcile(source, null); + + assertTrue(control.isNoUpdate()); + } + + @Test + void newVersionTriggersOneWorldIngestPerConfiguredWorld() { + WorldSource source = source("survival-source"); + source.getSpec().getWorlds().get(0).setName("world"); + WorldSource.WorldSelector second = new WorldSource.WorldSelector(); + second.setName("creative"); + source.getSpec().getWorlds().add(second); + + SourceVersion v1 = new SourceVersion("v1.zip", "v1.zip", Instant.parse("2026-08-09T10:00:00Z"), 100); + WorldSourceReconciler reconciler = reconciler(fixedVersions(List.of(v1))); + + reconciler.reconcile(source, null); + + List ingests = client.resources(WorldIngest.class) + .inNamespace("bluemap-friends") + .list() + .getItems(); + assertEquals(2, ingests.size(), "one WorldIngest per configured world"); + assertEquals("v1.zip", source.getStatus().getLastSeenVersion()); + assertEquals(WorldSourceReconciler.INGEST_TRIGGERED_REASON, readyReason(source)); + for (WorldIngest ingest : ingests) { + assertEquals("v1.zip", ingest.getSpec().getSourceVersion()); + assertEquals("survival-source", ingest.getSpec().getSourceRef().getName()); + } + } + + @Test + void alreadySeenVersionDoesNotTriggerAnotherIngest() { + WorldSource source = source("survival-source"); + SourceVersion v1 = new SourceVersion("v1.zip", "v1.zip", Instant.parse("2026-08-09T10:00:00Z"), 100); + WorldSourceReconciler reconciler = reconciler(fixedVersions(List.of(v1))); + + reconciler.reconcile(source, null); + int afterFirst = client.resources(WorldIngest.class) + .inNamespace("bluemap-friends") + .list() + .getItems() + .size(); + + // Simulate the next scheduled reconcile: due again, same latest version reported. + source.getStatus().setLastPollTime(Instant.parse("2026-08-09T11:00:00Z").toString()); + reconciler.reconcile(source, null); + int afterSecond = client.resources(WorldIngest.class) + .inNamespace("bluemap-friends") + .list() + .getItems() + .size(); + + assertEquals(1, afterFirst); + assertEquals(1, afterSecond, "no new WorldIngest for a version already seen"); + assertEquals(WorldSourceReconciler.UP_TO_DATE_REASON, readyReason(source)); + } + + @Test + void reconcilingTwiceForTheSameNewVersionIsIdempotent() { + WorldSource source = source("survival-source"); + SourceVersion v1 = new SourceVersion("v1.zip", "v1.zip", Instant.parse("2026-08-09T10:00:00Z"), 100); + WorldSourceReconciler reconciler = reconciler(fixedVersions(List.of(v1))); + + // First call creates the WorldIngest but crashes (simulated) before status is + // persisted -- re-running against the same in-memory object must not create a second + // WorldIngest for the same (source, world, version) triple. + reconciler.reconcile(source, null); + reconciler.reconcile(source, null); + + List ingests = client.resources(WorldIngest.class) + .inNamespace("bluemap-friends") + .list() + .getItems(); + assertEquals(1, ingests.size()); + } + + @Test + void noWorldsConfiguredIsReportedInsteadOfSilentlyDoingNothing() { + WorldSource source = source("s1"); + source.getSpec().getWorlds().clear(); + + reconciler(fixedVersions(List.of())).reconcile(source, null); + + assertEquals(WorldSourceReconciler.NO_WORLDS_CONFIGURED_REASON, readyReason(source)); + } + + @Test + void discoveryFailureIsReportedAndDoesNotCrashReconciliation() { + WorldSource source = source("s1"); + WorldSourceConnector failing = new WorldSourceConnector() { + @Override + public String type() { + return "s3"; + } + + @Override + public List discover(Map config) { + throw new RuntimeException("connection refused"); + } + + @Override + public void fetch(Map config, SourceVersion version, java.nio.file.Path workDir) {} + }; + + reconciler(failing).reconcile(source, null); + + assertEquals(WorldSourceReconciler.DISCOVERY_FAILED_REASON, readyReason(source)); + } + + @Test + void ingestNameCollisionWithAForeignResourceIsReportedAsAConflict() { + WorldSource source = source("survival-source"); + SourceVersion v1 = new SourceVersion("v1.zip", "v1.zip", Instant.parse("2026-08-09T10:00:00Z"), 100); + String collidingName = WorldSourceReconciler.ingestNameFor("survival-source", "world", "v1.zip"); + + WorldIngest foreign = new WorldIngest(); + foreign.setMetadata(new ObjectMetaBuilder() + .withName(collidingName) + .withNamespace("bluemap-friends") + .build()); // no owning labels at all + client.resources(WorldIngest.class).inNamespace("bluemap-friends").resource(foreign).create(); + + reconciler(fixedVersions(List.of(v1))).reconcile(source, null); + + assertEquals(WorldSourceReconciler.RESOURCE_CONFLICT_REASON, readyReason(source)); + } + + @Test + void discoverIsNeverCalledForAPushTypeSource() { + WorldSource source = source("s1"); + source.getSpec().setType("push"); + source.getSpec().setPoll(null); + + assertFalse(source.getSpec().getWorlds().isEmpty()); + WorldSourceReconciler reconciler = reconciler(fixedVersions(List.of())); + var control = reconciler.reconcile(source, null); + + assertNotNull(control); + assertEquals(WorldSourceReconciler.MANUAL_ONLY_REASON, readyReason(source)); + } + + @Test + void ingestNameForIsDeterministicAndStable() { + String first = WorldSourceReconciler.ingestNameFor("survival-source", "world", "v1.zip"); + String second = WorldSourceReconciler.ingestNameFor("survival-source", "world", "v1.zip"); + + assertEquals(first, second); + assertTrue(first.matches("[a-z0-9]([-a-z0-9]*[a-z0-9])?"), "must be a valid Kubernetes resource name: " + first); + } + + @Test + void ingestNameForDiffersByVersionEvenAfterSanitisation() { + // These two version ids sanitise to the exact same string, but the hash suffix + // (computed over the original, unsanitised id) must still keep them distinct. + String a = WorldSourceReconciler.ingestNameFor("s", "w", "v1/backup.zip"); + String b = WorldSourceReconciler.ingestNameFor("s", "w", "v1-backup.zip"); + + assertFalse(a.equals(b), "different raw version ids must never collide into the same ingest name"); + } +} diff --git a/operator/src/test/java/net/onelitefeather/apus/operator/render/BlueMapRenderReconcilerTest.java b/operator/src/test/java/net/onelitefeather/apus/operator/render/BlueMapRenderReconcilerTest.java index 847259c..4fbc97d 100644 --- a/operator/src/test/java/net/onelitefeather/apus/operator/render/BlueMapRenderReconcilerTest.java +++ b/operator/src/test/java/net/onelitefeather/apus/operator/render/BlueMapRenderReconcilerTest.java @@ -407,6 +407,28 @@ void transitionsToFailedWhenItsOwnJobExhaustsRetries() { assertEquals("JobFailed", readyReason(render)); } + /** + * B2: {@code backoffLimit} exists precisely so a transient pod failure gets retried -- a + * single failed *pod attempt* ({@code status.failed > 0}, no {@code Failed} condition yet) + * must not end the render terminally while the Job controller is still going to retry it. + * Same underlying mistake as {@code WorldIngestReconciler}'s twin fix. + */ + @Test + void aSingleFailedPodAttemptDoesNotEndTheRenderWhileTheJobStillHasRetriesLeft() { + client.resources(BlueMapMap.class).inNamespace("bluemap-friends").resource(boundMap()).create(); + BlueMapRenderReconciler reconciler = new BlueMapRenderReconciler(client, OperatorConfig.defaults()); + BlueMapRender render = renderWithUid("render-1"); + reconciler.reconcile(render, null); + + Job job = client.batch().v1().jobs().inNamespace("bluemap-friends").withName("render-1").get(); + job.setStatus(new JobStatusBuilder().withFailed(1).build()); + client.batch().v1().jobs().inNamespace("bluemap-friends").resource(job).updateStatus(); + + reconciler.reconcile(render, null); + + assertEquals("Rendering", render.getStatus().getPhase(), "a single failed attempt must not be terminal"); + } + // --- Progress transfer from the pod ------------------------------------------------------ @Test diff --git a/operator/src/test/java/net/onelitefeather/apus/operator/render/RenderJobBuilderTest.java b/operator/src/test/java/net/onelitefeather/apus/operator/render/RenderJobBuilderTest.java index 3e18fa2..f9d6756 100644 --- a/operator/src/test/java/net/onelitefeather/apus/operator/render/RenderJobBuilderTest.java +++ b/operator/src/test/java/net/onelitefeather/apus/operator/render/RenderJobBuilderTest.java @@ -149,7 +149,16 @@ void omitsTheOptionalPrefixVariableWhenTheMapStorageHasNone() { @Test void placesTheContainerImageFromTheOperatorConfig() { - OperatorConfig config = new OperatorConfig("rook-ceph-fr01", "feather-s3", "ceph-bucket-fr01", "apus/runner:1.2.3"); + OperatorConfig config = new OperatorConfig( + "rook-ceph-fr01", + "feather-s3", + "ceph-bucket-fr01", + "apus/runner:1.2.3", + "apus/ingest:dev", + "apus-bundles", + "http://rgw.rook-ceph-fr01.svc:80", + "us-east-1", + "apus-bundle-credentials"); Job job = RenderJobBuilder.build(render(), map(), "bucket-secret", config); diff --git a/runner/build.gradle.kts b/runner/build.gradle.kts index 21a6ee4..df4de08 100644 --- a/runner/build.gradle.kts +++ b/runner/build.gradle.kts @@ -10,6 +10,13 @@ dependencies { testImplementation(libs.testcontainers.minio) testImplementation("org.slf4j:slf4j-simple:2.0.16") + + // IngestRenderContractTest drives the real net.onelitefeather.apus.ingest.IngestMain entry + // point in-process (against the same MinIO Testcontainers instance the render half of that + // test already needs) to prove the phase 2b contract: a bundle the ingest module writes is + // exactly what this module's render image reads. Same "depend on the module rather than + // duplicating its logic" reasoning operator/build.gradle.kts already applies to :ingest. + testImplementation(project(":ingest")) } // Every test in this module is a container-based integration test: each starts MinIO plus diff --git a/runner/src/test/java/net/onelitefeather/apus/runner/IngestRenderContractTest.java b/runner/src/test/java/net/onelitefeather/apus/runner/IngestRenderContractTest.java new file mode 100644 index 0000000..5012b88 --- /dev/null +++ b/runner/src/test/java/net/onelitefeather/apus/runner/IngestRenderContractTest.java @@ -0,0 +1,375 @@ +/** + * Apus - render and host BlueMap maps on Kubernetes. + * Copyright (C) 2026 OneLiteFeather and contributors + *

+ * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + *

+ * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package net.onelitefeather.apus.runner; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import net.onelitefeather.apus.ingest.BundleManifest; +import net.onelitefeather.apus.ingest.IngestConfig; +import net.onelitefeather.apus.ingest.IngestMain; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.MinIOContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.Wait; + +/** + * Proves the whole phase 2b claim end to end: a world in Bukkit layout, run through the real + * {@link IngestMain} entry point against real MinIO, produces a bundle that the real {@code + * apus/runner} image (phase 1) can render without knowing anything about where the world data + * came from -- the one property the whole ingest layer exists to deliver (see the phase 2b plan's + * "Goal"). {@link RenderEndToEndTest} already proves the render half in isolation by seeding a + * bundle-shaped fixture directly with {@code mc mirror}; this test instead produces that bundle + * with the real ingest code path and only then hands it to the same render container, so a + * mismatch between what {@link IngestMain} writes and what {@code runner/bin/bundle-sync.sh} + * expects to read would show up here even if it would not show up in either half tested alone. + * + *

Reuses {@link MinioFixtures}'s MinIO/network/runner-container machinery rather than + * duplicating it -- see that class's Javadoc. + * + *

Requires the runner image to be built beforehand: + * {@code docker build -f runner/Dockerfile -t apus/runner:dev .} + */ +class IngestRenderContractTest { + + private static final String SOURCE_BUCKET = "ingest-sources"; + private static final String SOURCE_PREFIX = "raw/demo/"; + private static final String SOURCE_KEY = "v1.zip"; + + private static final String BUNDLE_TENANT = "acme"; + private static final String BUNDLE_WORLD_ID = "spawn"; + private static final String BUNDLE_VERSION = "v1"; + private static final String BUNDLE_PATH = BUNDLE_TENANT + "/" + BUNDLE_WORLD_ID + "/" + BUNDLE_VERSION; + + // What LayoutDetector.detect must normalise a Bukkit-layout source's sibling folders + // (world, world_nether, world_the_end) to -- the "core of normalisation" the phase 2b plan + // calls out explicitly: the same three logical names a vanilla source would also produce. + private static final List LOGICAL_DIMENSIONS = List.of("overworld", "the_nether", "the_end"); + private static final List REGION_FILE_NAMES = List.of("r.0.0.mca", "r.0.1.mca"); + + private static final String MAP_PREFIX = "ingest-e2e"; + private static final String RENDERED_TILE_KEY = MAP_PREFIX + "/overworld/tiles/0/x0/z0.prbm.gz"; + + private static final Pattern LS_JSON_KEY = Pattern.compile("\"key\":\"([^\"]*)\""); + private static final Pattern LS_JSON_LAST_MODIFIED = Pattern.compile("\"lastModified\":\"([^\"]*)\""); + + @Test + void bukkitLayoutWorldIngestedThenRenderedThroughTheRealRunnerImage(@TempDir Path tempDir) throws Exception { + Path zipFile = tempDir.resolve(SOURCE_KEY); + buildBukkitLayoutSourceZip(zipFile); + + try (Network network = Network.newNetwork(); + MinIOContainer minio = MinioFixtures.startMinio(network)) { + + seedSourceAndDestinationBuckets(network, zipFile); + + int exitCode = runIngest(minio, tempDir.resolve("work")); + assertEquals(0, exitCode, "ingest must succeed against the seeded Bukkit-layout source"); + + BundleManifest manifest = fetchAndParseManifest(network, tempDir.resolve("manifest-out")); + assertManifestIsComplete(manifest); + assertRealBucketListingMatchesTheManifestWithManifestWrittenLast(network); + + String overworldPath = manifest.dimensions().stream() + .filter(d -> d.id().equals("overworld")) + .findFirst() + .orElseThrow() + .path(); + String bundleUrl = "s3://" + MinioFixtures.WORLD_BUCKET + "/" + overworldPath; + + renderBundleAndVerifyATileLanded(network, bundleUrl); + } + } + + /** + * Builds one archive object containing a Bukkit-layout world -- {@code world/region}, + * {@code world_nether/DIM-1/region}, {@code world_the_end/DIM1/region} as sibling folders -- + * out of {@code testdata/mini-world}'s real region files, exactly as {@link + * net.onelitefeather.apus.ingest.connector.S3SourceConnector} requires: one fetchable object + * per version, unpacked if it is a recognised archive. The fixture itself only has an + * overworld; the nether/end folders reuse the same two region files, since only the + * directory names (not their contents) matter for proving layout normalisation. + */ + private static void buildBukkitLayoutSourceZip(Path zipFile) throws IOException { + Path region = MinioFixtures.fixture().resolve("region"); + byte[] regionZeroZero = Files.readAllBytes(region.resolve("r.0.0.mca")); + byte[] regionZeroOne = Files.readAllBytes(region.resolve("r.0.1.mca")); + byte[] levelDat = Files.readAllBytes(MinioFixtures.fixture().resolve("level.dat")); + + try (OutputStream fileOut = Files.newOutputStream(zipFile); + ZipOutputStream zip = new ZipOutputStream(fileOut)) { + writeZipEntry(zip, "world/level.dat", levelDat); + writeZipEntry(zip, "world/region/r.0.0.mca", regionZeroZero); + writeZipEntry(zip, "world/region/r.0.1.mca", regionZeroOne); + writeZipEntry(zip, "world_nether/DIM-1/region/r.0.0.mca", regionZeroZero); + writeZipEntry(zip, "world_nether/DIM-1/region/r.0.1.mca", regionZeroOne); + writeZipEntry(zip, "world_the_end/DIM1/region/r.0.0.mca", regionZeroZero); + writeZipEntry(zip, "world_the_end/DIM1/region/r.0.1.mca", regionZeroOne); + } + } + + private static void writeZipEntry(ZipOutputStream zip, String name, byte[] content) throws IOException { + zip.putNextEntry(new ZipEntry(name)); + zip.write(content); + zip.closeEntry(); + } + + /** Creates the source/bundle/map buckets and uploads the Bukkit-layout archive as one object. */ + private static void seedSourceAndDestinationBuckets(Network network, Path zipFile) { + try (GenericContainer seeder = MinioFixtures.mcContainer( + network, + "mc alias set m http://minio:9000 " + MinioFixtures.ACCESS_KEY + " " + + MinioFixtures.SECRET_KEY + " >/dev/null" + + " && mc mb --ignore-existing m/" + SOURCE_BUCKET + + " && mc mb --ignore-existing m/" + MinioFixtures.WORLD_BUCKET + + " && mc mb --ignore-existing m/" + MinioFixtures.MAP_BUCKET + + " && mc cp /source/" + SOURCE_KEY + " m/" + SOURCE_BUCKET + "/" + SOURCE_PREFIX + + SOURCE_KEY + + " && echo SEEDED") + .withFileSystemBind(zipFile.getParent().toString(), "/source", BindMode.READ_ONLY) + .waitingFor(Wait.forLogMessage(".*SEEDED.*", 1).withStartupTimeout(Duration.ofMinutes(2)))) { + seeder.start(); + } + } + + /** + * Drives the real ingest entry point in-process against the MinIO container's host-mapped + * port -- the same way {@code S3SourceConnectorTest} already talks to MinIO directly rather + * than through a container, since the code under test here isn't the thing running inside a + * container (that's {@code runner}, exercised separately below). + */ + private static int runIngest(MinIOContainer minio, Path workDir) { + Map env = new LinkedHashMap<>(); + env.put(IngestConfig.ENV_SOURCE_TYPE, "s3"); + env.put(IngestConfig.ENV_WORLD_NAME, "world"); + env.put(IngestConfig.ENV_LAYOUT, "auto"); + // The connector computes the fetch key as prefix + version id -- see + // S3SourceConnector.fetch -- so the version id must be relative to the prefix, not the + // prefixed key itself. + env.put(IngestConfig.ENV_SOURCE_VERSION, SOURCE_KEY); + env.put(IngestConfig.ENV_BUNDLE_BUCKET, MinioFixtures.WORLD_BUCKET); + env.put(IngestConfig.ENV_BUNDLE_TENANT, BUNDLE_TENANT); + env.put(IngestConfig.ENV_BUNDLE_WORLD_ID, BUNDLE_WORLD_ID); + env.put(IngestConfig.ENV_BUNDLE_VERSION, BUNDLE_VERSION); + env.put(IngestConfig.ENV_S3_ENDPOINT, minio.getS3URL()); + env.put(IngestConfig.ENV_S3_ACCESS_KEY, MinioFixtures.ACCESS_KEY); + env.put(IngestConfig.ENV_S3_SECRET_KEY, MinioFixtures.SECRET_KEY); + env.put(IngestConfig.ENV_SOURCE_S3_BUCKET, SOURCE_BUCKET); + env.put(IngestConfig.ENV_SOURCE_S3_PREFIX, SOURCE_PREFIX); + env.put(IngestConfig.ENV_SOURCE_S3_ENDPOINT, minio.getS3URL()); + env.put(IngestConfig.ENV_SOURCE_S3_ACCESS_KEY, MinioFixtures.ACCESS_KEY); + env.put(IngestConfig.ENV_SOURCE_S3_SECRET_KEY, MinioFixtures.SECRET_KEY); + + return IngestMain.run(env, workDir); + } + + private static BundleManifest fetchAndParseManifest(Network network, Path outDir) throws IOException { + Files.createDirectories(outDir); + try (GenericContainer fetcher = MinioFixtures.mcContainer( + network, + "mc alias set m http://minio:9000 " + MinioFixtures.ACCESS_KEY + " " + + MinioFixtures.SECRET_KEY + " >/dev/null" + + " && mc cat m/" + MinioFixtures.WORLD_BUCKET + "/" + BUNDLE_PATH + + "/manifest.json > /out/manifest.json" + + " && echo MANIFEST_FETCHED") + .withFileSystemBind(outDir.toString(), "/out", BindMode.READ_WRITE) + .waitingFor(Wait.forLogMessage(".*MANIFEST_FETCHED.*", 1).withStartupTimeout(Duration.ofMinutes(2)))) { + fetcher.start(); + } + String json = Files.readString(outDir.resolve("manifest.json")); + return BundleManifest.fromJson(json); + } + + /** + * Every claim task 7 must prove about the manifest itself: it is complete (all three logical + * dimensions, from a source that only had Bukkit sibling folders -- the normalisation the + * plan calls the "core of the ETL layer"), and the region list matches what was actually + * asked to be written (two region files, {@code r.0.0}/{@code r.0.1}, per dimension). + */ + private static void assertManifestIsComplete(BundleManifest manifest) { + assertEquals(1, manifest.schemaVersion()); + assertEquals(BUNDLE_TENANT, manifest.tenant()); + assertEquals(BUNDLE_WORLD_ID, manifest.worldId()); + assertEquals(BUNDLE_VERSION, manifest.version()); + assertEquals("s3", manifest.source().type()); + assertEquals( + "bukkit", + manifest.source().detectedLayout(), + "a source with world/world_nether/world_the_end siblings must be detected as bukkit"); + assertTrue(manifest.sizeBytes() > 0); + assertEquals("SHA-256", manifest.checksums().algorithm()); + assertFalse(manifest.checksums().manifest().isBlank()); + + Set dimensionIds = new LinkedHashSet<>(); + for (BundleManifest.DimensionInfo dimension : manifest.dimensions()) { + dimensionIds.add(dimension.id()); + assertEquals( + BUNDLE_PATH + "/dimensions/" + dimension.id(), + dimension.path(), + "dimension path must follow the bundle layout runner/bin/bundle-sync.sh expects"); + assertEquals(2, dimension.regionCount()); + Set regionCoords = new LinkedHashSet<>(); + for (int[] region : dimension.regions()) { + regionCoords.add(region[0] + "," + region[1]); + } + assertEquals( + Set.of("0,0", "0,1"), + regionCoords, + "region list must match the two .mca files actually present in the source"); + } + assertEquals( + new LinkedHashSet<>(LOGICAL_DIMENSIONS), + dimensionIds, + "Bukkit sibling folders must normalise to the same logical dimension names a vanilla " + + "layout would produce"); + } + + /** + * Independently cross-checks the manifest's claims against what MinIO actually holds: exactly + * the expected six region objects plus the manifest itself, no more, no less, and -- read + * straight from real object timestamps, not from a fake client's call log the way {@code + * BundleWriterTest} already proves this in isolation -- the manifest is the object with the + * latest {@code lastModified} of the bundle, i.e. it really was written last against a real + * S3-compatible store, not merely in a unit test double. + */ + private static void assertRealBucketListingMatchesTheManifestWithManifestWrittenLast(Network network) { + Set expectedKeys = new LinkedHashSet<>(); + expectedKeys.add("manifest.json"); + for (String dimension : LOGICAL_DIMENSIONS) { + for (String regionFile : REGION_FILE_NAMES) { + expectedKeys.add("dimensions/" + dimension + "/region/" + regionFile); + } + } + + String logs; + try (GenericContainer lister = MinioFixtures.mcContainer( + network, + "mc alias set m http://minio:9000 " + MinioFixtures.ACCESS_KEY + " " + + MinioFixtures.SECRET_KEY + " >/dev/null" + + " && mc ls --recursive --json m/" + MinioFixtures.WORLD_BUCKET + "/" + BUNDLE_PATH + + "/" + + " && echo LS_DONE") + .waitingFor(Wait.forLogMessage(".*LS_DONE.*", 1).withStartupTimeout(Duration.ofMinutes(2)))) { + lister.start(); + logs = lister.getLogs(); + } + + Map lastModifiedByKey = new LinkedHashMap<>(); + for (String line : logs.split("\\R")) { + if (!line.startsWith("{")) { + continue; // not an mc ls --json line (e.g. the LS_DONE marker) + } + Matcher keyMatcher = LS_JSON_KEY.matcher(line); + Matcher lastModifiedMatcher = LS_JSON_LAST_MODIFIED.matcher(line); + if (!keyMatcher.find() || !lastModifiedMatcher.find()) { + continue; + } + lastModifiedByKey.put(keyMatcher.group(1), Instant.parse(lastModifiedMatcher.group(1))); + } + + assertEquals(expectedKeys, lastModifiedByKey.keySet(), "bucket must hold exactly the bundle's own objects:\n" + logs); + + Instant manifestWrittenAt = lastModifiedByKey.get("manifest.json"); + for (Map.Entry entry : lastModifiedByKey.entrySet()) { + if (entry.getKey().equals("manifest.json")) { + continue; + } + assertFalse( + manifestWrittenAt.isBefore(entry.getValue()), + "manifest.json (" + manifestWrittenAt + ") must not be older than " + entry.getKey() + " (" + + entry.getValue() + ") -- the manifest is the bundle's commit point and must be " + + "written last"); + } + } + + /** + * The actual proof that the ingest/render contract holds: starts the real {@code apus/runner} + * image (phase 1) against the bundle {@link IngestMain} just wrote, using exactly the {@code + * bundleUrl} a {@code BlueMapRender} would carry -- {@code s3:///} -- + * and checks a real rendered tile lands in the map bucket, not merely that the render + * container exited 0. + */ + private static void renderBundleAndVerifyATileLanded(Network network, String bundleUrl) { + String image = System.getProperty("apus.runner.image", "apus/runner:dev"); + + try (GenericContainer runner = MinioFixtures.runnerContainer(network, image) + .withEnv("APUS_WORLD_S3_URL", bundleUrl) + .withEnv("APUS_MAP_PREFIX", MAP_PREFIX) + .withEnv("APUS_RENDER_THREADS", "2") + .waitingFor(Wait.forLogMessage(".*starting BlueMap.*", 1).withStartupTimeout(Duration.ofMinutes(5)))) { + + runner.start(); + + long deadline = System.currentTimeMillis() + Duration.ofMinutes(15).toMillis(); + while (runner.isRunning() && System.currentTimeMillis() < deadline) { + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + assertFalse(runner.isRunning(), "render container must exit after rendering, it must not hang"); + Long exitCode = runner.getCurrentContainerInfo().getState().getExitCodeLong(); + assertEquals( + 0L, + exitCode, + "BlueMap CLI must exit 0 rendering the bundle the ingest module just wrote; logs:\n" + + runner.getLogs()); + } + + try (GenericContainer verifier = MinioFixtures.mcContainer( + network, + "mc alias set m http://minio:9000 " + MinioFixtures.ACCESS_KEY + " " + + MinioFixtures.SECRET_KEY + + " && COUNT=$(mc ls --recursive m/" + MinioFixtures.MAP_BUCKET + " | wc -l)" + + " && echo OBJECTS=$COUNT" + + " ; (mc stat m/" + MinioFixtures.MAP_BUCKET + "/" + RENDERED_TILE_KEY + + " >/dev/null 2>&1 && echo TILE_FOUND=yes || echo TILE_FOUND=no)") + .waitingFor(Wait.forLogMessage(".*TILE_FOUND=.*", 1).withStartupTimeout(Duration.ofMinutes(2)))) { + verifier.start(); + String logs = verifier.getLogs(); + assertTrue(logs.contains("OBJECTS="), logs); + assertFalse(logs.contains("OBJECTS=0"), "map bucket must not be empty after a render:\n" + logs); + assertTrue( + logs.contains("TILE_FOUND=yes"), + "expected a real render tile at " + MinioFixtures.MAP_BUCKET + "/" + RENDERED_TILE_KEY + + "; logs:\n" + logs); + } + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index ebff1aa..1110148 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,6 +1,6 @@ rootProject.name = "Apus" -include("telemetry-addon", "runner", "operator") +include("telemetry-addon", "runner", "operator", "ingest") dependencyResolutionManagement { repositories { @@ -17,6 +17,39 @@ dependencyResolutionManagement { version("shadow", "9.3.2") version("josdk", "5.5.1") version("fabric8", "7.8.0") + // Jackson: not a new dependency family for the project -- fabric8's kubernetes-client + // already pulls jackson-databind transitively for the operator module -- just the + // first place it's declared explicitly, for the ingest module's BundleManifest + // serialisation and Pterodactyl JSON parsing (replacing two hand-rolled JSON + // parsers). Version verified against Maven Central on 2026-08-08: 2.22.1 is the + // newest jackson-bom release (2.22.2 does not exist yet). Note that jackson-bom + // 2.20+ pins jackson-annotations to a patch-less "2.22" version, by design (see the + // bom's own POM comment) -- only jackson-databind itself needs a version("jackson") + // reference here. + version("jackson", "2.22.1") + // AWS SDK v2, not the MinIO Java client: runner/vendor/BlueMapS3Storage.jar (the + // BlueMap storage addon the render container already uses) is itself built on + // software.amazon.nio.spi.s3, which wraps this same SDK. Using it here too keeps + // exactly one S3 client family/credential-provider chain across the project + // instead of two competing ones, and it works against any S3-compatible endpoint + // (Rook/Ceph, MinIO, R2, ...) via endpoint override + path-style access -- nothing + // MinIO-specific is needed. Version verified against Maven Central on 2026-08-08. + version("aws-sdk", "2.46.7") + + // cron-utils: parses/evaluates the Cron expression in WorldSourceSpec.poll + // (phase 2b, task 6). Chosen over hand-rolling a parser (an explicitly named + // known error source in the task brief) and over pulling in a full scheduler + // framework (Quartz, Spring) just for "is this cron string due yet" -- this + // operator never runs cron jobs itself, JOSDK's own reschedule mechanism does + // that; only expression parsing + next-execution-time math is needed, which is + // exactly cron-utils' scope. Verified against Maven Central on 2026-08-09: 9.2.1 + // is the newest release (last published 2023-03, no newer version exists). Its + // POM (also checked directly) has exactly one non-test runtime dependency, + // slf4j-api (compile scope) -- javax.validation:validation-api is "provided" + // (only needed if bean-validation annotations are actually exercised, which + // CronParser/ExecutionTime do not do), so this stays the "slim library" the + // brief asks for rather than a heavyweight addition. + version("cron-utils", "9.2.1") library("bluemap.api", "de.bluecolored", "bluemap-api").versionRef("bluemap-api") library("bluemap.core", "de.bluecolored", "bluemap-core").versionRef("bluemap") @@ -40,6 +73,14 @@ dependencyResolutionManagement { // (that one targets tests against a real cluster and ships no mock classes). library("fabric8.server.mock", "io.fabric8", "kubernetes-server-mock").versionRef("fabric8") + library("aws.sdk.bom", "software.amazon.awssdk", "bom").versionRef("aws-sdk") + library("aws.sdk.s3", "software.amazon.awssdk", "s3").withoutVersion() + + library("jackson.bom", "com.fasterxml.jackson", "jackson-bom").versionRef("jackson") + library("jackson.databind", "com.fasterxml.jackson.core", "jackson-databind").withoutVersion() + + library("cron.utils", "com.cronutils", "cron-utils").versionRef("cron-utils") + plugin("spotless", "com.diffplug.spotless").versionRef("spotless") plugin("shadow", "com.gradleup.shadow").versionRef("shadow") }