diff --git a/docs/cui-marking.md b/docs/cui-marking.md index 88996bae..856f9ae1 100644 --- a/docs/cui-marking.md +++ b/docs/cui-marking.md @@ -16,6 +16,22 @@ Example: `list packages --json` would otherwise write `packages.json`. The status code alone decides the outcome. Marked content is always classified: there is no unclassified artifact. Any other answer, whether a **204**, an unexpected status, a transport failure, or a **200** without a usable cover page, aborts the command and leaves no output behind. +A command asks once and applies the same outcome to everything it writes, so an export made up of several files cannot come out partly marked. + +## Caching + +The first successful answer is cached and reused by later commands, so a shell session asks the endpoint once rather than once per artifact. + +| Aspect | Behaviour | +|---|---| +| Lifetime | Until the shell session ends or the machine restarts. There is no time limit within a session. | +| Location | A file in the system temp directory, readable only by the current user. Set `CONTENT_CLI_CUI_CACHE_DIR` to place it elsewhere, for example one directory per CI job. | +| Keyed by | Profile name, team URL, and the shell session, so switching profile, team, or terminal fetches again. | +| Failures | Never cached. The command errors, and the next one asks again. | +| Unreadable entry | Discarded and refetched, never treated as "not classified". | + +Because a decision survives for the whole session, a change to the team's CUI settings takes effect for the current shell only after the cached answer is dropped. Open a new terminal, or delete the cache file, to pick it up right away. + ## Scope: how the write is triggered | Trigger | Commands | Example when classified | diff --git a/src/core/command/cli-context.ts b/src/core/command/cli-context.ts index 1aafd622..e4d977b0 100644 --- a/src/core/command/cli-context.ts +++ b/src/core/command/cli-context.ts @@ -4,6 +4,7 @@ import {FatalError, logger} from "../utils/logger"; import {Profile} from "../profile/profile.interface"; import { GitProfileService } from "../git-profile/git-profile.service"; import { GitProfile } from "../git-profile/git-profile.interface"; +import type { CuiMarkingDecision } from "../utils/cui-api"; /** * The execution context object is passed to the modules to access @@ -16,6 +17,7 @@ export class Context { public _httpClient: HttpClient; public profile: Profile; public gitProfile: GitProfile; + public cuiMarking: Promise | undefined; private log = logger; private profileName: string | undefined; diff --git a/src/core/utils/cui-api.ts b/src/core/utils/cui-api.ts index 9011c568..0065fceb 100644 --- a/src/core/utils/cui-api.ts +++ b/src/core/utils/cui-api.ts @@ -1,6 +1,6 @@ -import { HttpClient } from "../http/http-client"; import { FatalError, logger } from "./logger"; import { Context } from "../command/cli-context"; +import { CuiMarkingCache } from "./cui-marking-cache"; export interface CuiPdfCoverResponse { coverPage?: { pdfContent: string; encoding: string }; @@ -21,14 +21,57 @@ export class CuiApi { private static readonly STATUS_OK = 200; private static readonly STATUS_FORBIDDEN = 403; - private readonly httpClient: () => HttpClient; + private readonly context: Context; + private readonly cache: CuiMarkingCache; constructor(context: Context) { - this.httpClient = () => context.httpClient; + this.context = context; + this.cache = new CuiMarkingCache(context); } - public async getCuiMarking(): Promise { - const { status, data } = await this.httpClient().getStatusAndData(CuiApi.CUI_PDF_COVER_SHEET_URL); + public getCuiMarking(): Promise { + if (!this.context.cuiMarking) { + this.context.cuiMarking = this.resolveCuiMarking().catch(error => { + this.context.cuiMarking = undefined; + throw error; + }); + } + + return this.context.cuiMarking; + } + + private async resolveCuiMarking(): Promise { + const cached = this.readCachedMarking(); + if (cached) { + logger.debug("Reusing the CUI marking decision cached for this session"); + return cached; + } + + const decision = await this.fetchCuiMarking(); + this.cache.write(decision); + + return decision; + } + + private readCachedMarking(): CuiMarkingDecision | undefined { + const cached = this.cache.read() as CuiMarkingDecision | undefined; + + if (cached?.marking === CuiMarking.DISABLED) { + return cached; + } + if (cached?.marking === CuiMarking.CLASSIFIED && cached.cover) { + return cached; + } + + if (cached) { + logger.debug("Ignoring a cached CUI marking decision that cannot be used"); + } + + return undefined; + } + + private async fetchCuiMarking(): Promise { + const { status, data } = await this.context.httpClient.getStatusAndData(CuiApi.CUI_PDF_COVER_SHEET_URL); if (status === CuiApi.STATUS_FORBIDDEN) { logger.debug("CUI marking does not apply, the feature flag is disabled"); diff --git a/src/core/utils/cui-marking-cache.ts b/src/core/utils/cui-marking-cache.ts new file mode 100644 index 00000000..f87ed23f --- /dev/null +++ b/src/core/utils/cui-marking-cache.ts @@ -0,0 +1,81 @@ +import { createHash } from "node:crypto"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { Context } from "../command/cli-context"; +import { FileConstants } from "./file.constants"; +import { logger } from "./logger"; + +export class CuiMarkingCache { + public static readonly CACHE_DIRECTORY_ENV_VARIABLE = "CONTENT_CLI_CUI_CACHE_DIR"; + + private static readonly FILE_PREFIX = "content-cli-cui-marking-"; + + private readonly context: Context; + + constructor(context: Context) { + this.context = context; + } + + public read(): unknown { + const filePath = this.resolveFilePath(); + if (!filePath || !fs.existsSync(filePath)) { + return undefined; + } + + try { + return JSON.parse(fs.readFileSync(filePath, { encoding: "utf-8" })); + } catch (error) { + // The error is interpolated: passing it as metadata makes the logger exit the process. + logger.debug(`Discarding an unreadable CUI marking cache at ${filePath}: ${error}`); + this.clear(); + return undefined; + } + } + + public write(decision: unknown): void { + const filePath = this.resolveFilePath(); + if (!filePath) { + return; + } + + try { + fs.mkdirSync(path.dirname(filePath), { + recursive: true, + mode: FileConstants.DEFAULT_FOLDER_PERMISSIONS, + }); + fs.writeFileSync(filePath, JSON.stringify(decision), { + encoding: "utf-8", + mode: FileConstants.DEFAULT_FILE_PERMISSIONS, + }); + } catch (error) { + logger.debug(`Could not cache the CUI marking decision at ${filePath}: ${error}`); + } + } + + public clear(): void { + const filePath = this.resolveFilePath(); + if (!filePath) { + return; + } + + try { + fs.rmSync(filePath, { force: true }); + } catch (error) { + logger.debug(`Could not remove the CUI marking cache at ${filePath}: ${error}`); + } + } + + private resolveFilePath(): string | undefined { + const profile = this.context.profile; + if (!profile?.team) { + return undefined; + } + + const key = createHash("sha256").update(`${profile.name}|${profile.team}`).digest("hex").slice(0, 16); + const directory = process.env[CuiMarkingCache.CACHE_DIRECTORY_ENV_VARIABLE] || os.tmpdir(); + + // Tied to the parent shell so a new terminal starts over, and to the temp dir so a reboot clears it. + return path.join(directory, `${CuiMarkingCache.FILE_PREFIX}${key}-${process.ppid}.json`); + } +} diff --git a/tests/core/utils/cui-file-service.spec.ts b/tests/core/utils/cui-file-service.spec.ts index 826b9925..15ab3ffd 100644 --- a/tests/core/utils/cui-file-service.spec.ts +++ b/tests/core/utils/cui-file-service.spec.ts @@ -1,10 +1,13 @@ -import { accessSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { resolve } from "node:path"; +import { accessSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; import AdmZip = require("adm-zip"); +import { Context } from "../../../src/core/command/cli-context"; +import { HttpClient } from "../../../src/core/http/http-client"; import { CuiFileService } from "../../../src/core/utils/cui-file-service"; +import { CuiMarkingCache } from "../../../src/core/utils/cui-marking-cache"; import { FatalError } from "../../../src/core/utils/logger"; import { testContext } from "../../utls/test-context"; -import { mockAxiosGetError, mockAxiosGetWithStatus } from "../../utls/http-requests-mock"; +import { mockAxiosGetError, mockAxiosGetWithStatus, mockedAxiosInstance } from "../../utls/http-requests-mock"; describe("CuiFileService", () => { const COVER_URL = "https://myTeam.celonis.cloud/api/team/cui-settings/cui-pdf-cover"; @@ -22,6 +25,9 @@ describe("CuiFileService", () => { const readFile = (filename: string): Buffer => readFileSync(resolve(process.cwd(), filename)); + const coverRequestCount = (): number => + (mockedAxiosInstance.get as jest.Mock).mock.calls.filter(call => call[0] === COVER_URL).length; + beforeEach(() => { cuiFileService = new CuiFileService(testContext); }); @@ -138,6 +144,110 @@ describe("CuiFileService", () => { }); }); + describe("when several artifacts are written in the same run", () => { + it("Should ask for the cover once and mark every artifact the same way", async () => { + mockAxiosGetWithStatus(COVER_URL, 200, coverResponse()); + + const firstFilename = await cuiFileService.writeToFileWithGivenName(PAYLOAD, "packages.json"); + const secondFilename = await cuiFileService.writeToFileWithGivenName(PAYLOAD, "summary.json"); + + expect(firstFilename).toEqual("CUI - packages.zip"); + expect(secondFilename).toEqual("CUI - summary.zip"); + expect(coverRequestCount()).toEqual(1); + }); + + it("Should ask again after a failed cover request", async () => { + mockAxiosGetError(COVER_URL, 500, { message: "boom" }); + + await expect(cuiFileService.writeToFileWithGivenName(PAYLOAD, "failed.json")).rejects.toThrow(FatalError); + + mockAxiosGetWithStatus(COVER_URL, 200, coverResponse()); + const filename = await cuiFileService.writeToFileWithGivenName(PAYLOAD, "recovered.json"); + + expect(filename).toEqual("CUI - recovered.zip"); + expect(coverRequestCount()).toEqual(2); + }); + }); + + describe("when a later command runs in the same session", () => { + const cacheDirectory = (): string => process.env[CuiMarkingCache.CACHE_DIRECTORY_ENV_VARIABLE]; + + const nextRun = (profileName: string = "test"): CuiFileService => { + const context = new Context({}); + context.profile = { ...testContext.profile, name: profileName }; + context._httpClient = new HttpClient(context); + + return new CuiFileService(context); + }; + + const overwriteCachedDecision = (contents: string): void => + readdirSync(cacheDirectory()).forEach(entry => writeFileSync(join(cacheDirectory(), entry), contents)); + + it("Should reuse a classified decision without asking again", async () => { + mockAxiosGetWithStatus(COVER_URL, 200, coverResponse()); + await cuiFileService.writeToFileWithGivenName(PAYLOAD, "first-run.json"); + + const filename = await nextRun().writeToFileWithGivenName(PAYLOAD, "second-run.json"); + + expect(filename).toEqual("CUI - second-run.zip"); + expect(readFile(filename).length).toBeGreaterThan(0); + expect(coverRequestCount()).toEqual(1); + }); + + it("Should reuse a disabled decision without asking again", async () => { + mockAxiosGetError(COVER_URL, 403, { errorCode: "feature-disabled" }); + await cuiFileService.writeToFileWithGivenName(PAYLOAD, "first-run.json"); + + const filename = await nextRun().writeToFileWithGivenName(PAYLOAD, "second-run.json"); + + expect(filename).toEqual("second-run.json"); + expect(coverRequestCount()).toEqual(1); + }); + + it("Should ask again when the earlier command failed", async () => { + mockAxiosGetError(COVER_URL, 500, { message: "boom" }); + await expect(cuiFileService.writeToFileWithGivenName(PAYLOAD, "failed.json")).rejects.toThrow(FatalError); + + mockAxiosGetWithStatus(COVER_URL, 200, coverResponse()); + const filename = await nextRun().writeToFileWithGivenName(PAYLOAD, "recovered.json"); + + expect(filename).toEqual("CUI - recovered.zip"); + expect(coverRequestCount()).toEqual(2); + }); + + it("Should ask again when the cached decision is unreadable", async () => { + mockAxiosGetWithStatus(COVER_URL, 200, coverResponse()); + await cuiFileService.writeToFileWithGivenName(PAYLOAD, "first-run.json"); + overwriteCachedDecision("not json"); + + const filename = await nextRun().writeToFileWithGivenName(PAYLOAD, "after-corruption.json"); + + expect(filename).toEqual("CUI - after-corruption.zip"); + expect(coverRequestCount()).toEqual(2); + }); + + it("Should ask again when the cached decision is classified without a cover", async () => { + mockAxiosGetWithStatus(COVER_URL, 200, coverResponse()); + await cuiFileService.writeToFileWithGivenName(PAYLOAD, "first-run.json"); + overwriteCachedDecision(JSON.stringify({ marking: "CLASSIFIED" })); + + const filename = await nextRun().writeToFileWithGivenName(PAYLOAD, "after-tampering.json"); + + expect(filename).toEqual("CUI - after-tampering.zip"); + expect(coverRequestCount()).toEqual(2); + }); + + it("Should not reuse a decision made for another profile", async () => { + mockAxiosGetWithStatus(COVER_URL, 200, coverResponse()); + await cuiFileService.writeToFileWithGivenName(PAYLOAD, "first-run.json"); + + const filename = await nextRun("other-team").writeToFileWithGivenName(PAYLOAD, "other-profile.json"); + + expect(filename).toEqual("CUI - other-profile.zip"); + expect(coverRequestCount()).toEqual(2); + }); + }); + describe("when the artifact is a directory", () => { const writeTree = (targetDir: string): void => { mkdirSync(resolve(process.cwd(), targetDir, "nodes"), { recursive: true }); diff --git a/tests/core/utils/cui-marking-cache.spec.ts b/tests/core/utils/cui-marking-cache.spec.ts new file mode 100644 index 00000000..5e605009 --- /dev/null +++ b/tests/core/utils/cui-marking-cache.spec.ts @@ -0,0 +1,50 @@ +import { writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { Context } from "../../../src/core/command/cli-context"; +import { CuiMarkingCache } from "../../../src/core/utils/cui-marking-cache"; +import { testContext } from "../../utls/test-context"; + +describe("CuiMarkingCache", () => { + const DECISION = { marking: "DISABLED" }; + + const configuredDirectory = process.env[CuiMarkingCache.CACHE_DIRECTORY_ENV_VARIABLE]; + + afterEach(() => { + process.env[CuiMarkingCache.CACHE_DIRECTORY_ENV_VARIABLE] = configuredDirectory; + }); + + it("Should keep the decision between two instances", () => { + new CuiMarkingCache(testContext).write(DECISION); + + expect(new CuiMarkingCache(testContext).read()).toEqual(DECISION); + }); + + it("Should forget the decision once cleared", () => { + const cache = new CuiMarkingCache(testContext); + cache.write(DECISION); + + cache.clear(); + + expect(cache.read()).toBeUndefined(); + }); + + it("Should do nothing when the profile has no team", () => { + const cache = new CuiMarkingCache(new Context({})); + + cache.write(DECISION); + + expect(cache.read()).toBeUndefined(); + expect(() => cache.clear()).not.toThrow(); + }); + + it("Should stay quiet when the location cannot be written", () => { + const blockingFile = resolve(process.cwd(), "not-a-directory"); + writeFileSync(blockingFile, ""); + process.env[CuiMarkingCache.CACHE_DIRECTORY_ENV_VARIABLE] = join(blockingFile, "cache"); + + const cache = new CuiMarkingCache(testContext); + cache.write(DECISION); + + expect(cache.read()).toBeUndefined(); + }); +}); diff --git a/tests/jest.setup.ts b/tests/jest.setup.ts index 3bb469d9..f4c623b4 100644 --- a/tests/jest.setup.ts +++ b/tests/jest.setup.ts @@ -8,9 +8,19 @@ import { join } from "path"; import process = require("process"); import { rmTempDir } from "./utls/fs-utils"; +import { CuiMarkingCache } from "../src/core/utils/cui-marking-cache"; mockAxios(); +// Workers share a parent pid, so each needs its own CUI cache dir to stay independent. +const cuiCacheDir = fs.mkdtempSync(join(tmpdir(), "jest-cui-cache")); +process.env[CuiMarkingCache.CACHE_DIRECTORY_ENV_VARIABLE] = cuiCacheDir; + +// Removed wholesale rather than listed, because some specs spy on readdirSync. +afterEach(() => { + fs.rmSync(cuiCacheDir, { recursive: true, force: true }); +}); + let tempDir = null; beforeAll(done => { fs.mkdtemp(join(tmpdir(), "jest"), (err, dir) => { diff --git a/tests/utls/test-context.ts b/tests/utls/test-context.ts index e6cabcf6..b1c79276 100644 --- a/tests/utls/test-context.ts +++ b/tests/utls/test-context.ts @@ -10,4 +10,9 @@ testContext.profile = { authenticationType: "Bearer" } testContext._httpClient = new HttpClient(testContext); + +afterEach(() => { + testContext.cuiMarking = undefined; +}); + export { testContext };