From 1d3fa25389e68bc590868e4064c5e51cd54d80fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Mon, 24 Aug 2026 15:39:57 +0200 Subject: [PATCH 1/2] Allow enabling versioning on buckets with a GCP location constraint Arsenal now supports versioning on the GCP backend, so drop gcp from versioningNotImplBackends and stop naming it in the error messages. Issue: CLDSRV-984 --- constants.js | 2 +- lib/api/apiUtils/object/createAndStoreObject.js | 2 +- lib/api/bucketPutVersioning.js | 2 +- lib/api/objectCopy.js | 2 +- package.json | 2 +- tests/unit/api/bucketPutVersioning.js | 2 +- tests/unit/multipleBackend/VersioningBackendClient.js | 2 +- yarn.lock | 10 +++++----- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/constants.js b/constants.js index cdcd1c5d1a..334ca12fb2 100644 --- a/constants.js +++ b/constants.js @@ -153,7 +153,7 @@ const constants = { // for external backends, don't call unless at least 1 minute // (60,000 milliseconds) since last call externalBackendHealthCheckInterval: 60000, - versioningNotImplBackends: { azure: true, gcp: true }, + versioningNotImplBackends: { azure: true }, mpuMDStoredExternallyBackend: { aws_s3: true, gcp: true }, skipBatchDeleteBackends: { azure: true, gcp: true }, s3HandledBackends: { azure: true, gcp: true }, diff --git a/lib/api/apiUtils/object/createAndStoreObject.js b/lib/api/apiUtils/object/createAndStoreObject.js index 2c69a55ccf..baaf4155aa 100644 --- a/lib/api/apiUtils/object/createAndStoreObject.js +++ b/lib/api/apiUtils/object/createAndStoreObject.js @@ -24,7 +24,7 @@ const { const { externalBackends, versioningNotImplBackends } = constants; const externalVersioningErrorMessage = - 'We do not currently support putting a versioned object to a location-constraint of type Azure or GCP.'; + 'We do not currently support putting a versioned object to a location-constraint of type Azure.'; /** * Validate and compute the checksum for a zero-size object body. diff --git a/lib/api/bucketPutVersioning.js b/lib/api/bucketPutVersioning.js index 5f872cc0dd..c5a0258d30 100644 --- a/lib/api/bucketPutVersioning.js +++ b/lib/api/bucketPutVersioning.js @@ -12,7 +12,7 @@ const { config } = require('../Config'); const monitoring = require('../utilities/monitoringHandler'); const externalVersioningErrorMessage = 'We do not currently support putting ' + -'a versioned object to a location-constraint of type Azure or GCP.'; +'a versioned object to a location-constraint of type Azure.'; const replicationVersioningErrorMessage = 'A replication configuration is ' + 'present on this bucket, so you cannot change the versioning state. To ' + diff --git a/lib/api/objectCopy.js b/lib/api/objectCopy.js index fa0b237ebf..9ffc35d685 100644 --- a/lib/api/objectCopy.js +++ b/lib/api/objectCopy.js @@ -39,7 +39,7 @@ const versionIdUtils = versioning.VersionID; const locationHeader = constants.objectLocationConstraintHeader; const versioningNotImplBackends = constants.versioningNotImplBackends; const externalVersioningErrorMessage = - 'We do not currently support putting a versioned object to a location-constraint of type AWS or Azure or GCP.'; + 'We do not currently support putting a versioned object to a location-constraint of type Azure.'; /** * Compute the prior data locations that are orphaned. diff --git a/package.json b/package.json index 0db3eff54b..5a48c63b68 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@opentelemetry/instrumentation-ioredis": "~0.64.0", "@opentelemetry/instrumentation-mongodb": "~0.69.0", "@smithy/node-http-handler": "^3.0.0", - "arsenal": "git+https://github.com/scality/arsenal#8.5.6", + "arsenal": "git+https://github.com/scality/arsenal#9d4bffafa667e3aafc31355ede3bf7af93d1930e", "async": "2.6.4", "aws-crt": "^1.24.0", "bucketclient": "scality/bucketclient#8.2.7", diff --git a/tests/unit/api/bucketPutVersioning.js b/tests/unit/api/bucketPutVersioning.js index 462fadcf74..9ec276b120 100644 --- a/tests/unit/api/bucketPutVersioning.js +++ b/tests/unit/api/bucketPutVersioning.js @@ -45,7 +45,7 @@ const xmlReplicationConfiguration = ''; const externalVersioningErrorMessage = 'We do not currently support putting ' + -'a versioned object to a location-constraint of type Azure or GCP.'; +'a versioned object to a location-constraint of type Azure.'; const log = new DummyRequestLogger(); const bucketName = 'bucketname'; diff --git a/tests/unit/multipleBackend/VersioningBackendClient.js b/tests/unit/multipleBackend/VersioningBackendClient.js index 60d8527602..bbed543876 100644 --- a/tests/unit/multipleBackend/VersioningBackendClient.js +++ b/tests/unit/multipleBackend/VersioningBackendClient.js @@ -105,7 +105,7 @@ describe('AwsClient::copyObject', () => { genTests.forEach(test => it(test.msg, done => { testClient._supportsVersioning = test.input.supportsVersioning; testClient._client.versioning = test.input.enableMockVersioning; - testClient.copyObject(copyObjectRequest, null, key, + testClient.copyObject(copyObjectRequest, null, key, undefined, sourceLocationConstraint, null, config, log, err => test.callback(err, done)); })); diff --git a/yarn.lock b/yarn.lock index 586c43f42b..732b605cc4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19,7 +19,7 @@ "@aws-sdk/types" "^3.222.0" tslib "^1.11.1" -"@aws-crypto/crc32@5.2.0", "@aws-crypto/crc32@^5.2.0": +"@aws-crypto/crc32@5.2.0": version "5.2.0" resolved "https://registry.yarnpkg.com/@aws-crypto/crc32/-/crc32-5.2.0.tgz#cfcc22570949c98c6689cfcbd2d693d36cdae2e1" integrity sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg== @@ -28,7 +28,7 @@ "@aws-sdk/types" "^3.222.0" tslib "^2.6.2" -"@aws-crypto/crc32c@5.2.0", "@aws-crypto/crc32c@^5.2.0": +"@aws-crypto/crc32c@5.2.0": version "5.2.0" resolved "https://registry.yarnpkg.com/@aws-crypto/crc32c/-/crc32c-5.2.0.tgz#4e34aab7f419307821509a98b9b08e84e0c1917e" integrity sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag== @@ -6596,9 +6596,9 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/arsenal#8.5.6": - version "8.5.6" - resolved "git+https://github.com/scality/arsenal#0db557930c7d13204167188a7503e6e00d154df5" +"arsenal@git+https://github.com/scality/arsenal#9d4bffafa667e3aafc31355ede3bf7af93d1930e": + version "8.5.12" + resolved "git+https://github.com/scality/arsenal#9d4bffafa667e3aafc31355ede3bf7af93d1930e" dependencies: "@aws-sdk/client-kms" "^3.975.0" "@aws-sdk/client-s3" "^3.975.0" From 5029db22f7c65ce0804e7609db740c5a59ada729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Fri, 28 Aug 2026 14:26:33 +0200 Subject: [PATCH 2/2] Ignore the request checksum when creating a delete marker A delete marker has no body, but createAndStoreObject took its checksum from the request headers, which on a multi-object delete cover the request XML. On an external location the mismatch failed the put, and the rollback error then hit an unguarded result.versionId in multiObjectDelete, killing the worker with a 502. Skip the header checksum for delete markers, and guard the result so any later error is returned as a proper S3 error. Issue: CLDSRV-984 --- .../apiUtils/object/createAndStoreObject.js | 4 +- lib/api/multiObjectDelete.js | 2 +- tests/unit/api/createAndStoreObject.js | 38 +++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/lib/api/apiUtils/object/createAndStoreObject.js b/lib/api/apiUtils/object/createAndStoreObject.js index baaf4155aa..37834b8435 100644 --- a/lib/api/apiUtils/object/createAndStoreObject.js +++ b/lib/api/apiUtils/object/createAndStoreObject.js @@ -309,7 +309,9 @@ function createAndStoreObject( } } - const headerChecksum = getChecksumDataFromHeaders(request.headers); + // A delete marker has no body: an x-amz-checksum-* header + // describes the request payload (e.g. a multi-object delete XML). + const headerChecksum = isDeleteMarker ? null : getChecksumDataFromHeaders(request.headers); if (headerChecksum && headerChecksum.error) { return next(arsenalErrorFromChecksumError(headerChecksum)); } diff --git a/lib/api/multiObjectDelete.js b/lib/api/multiObjectDelete.js index a512a591e9..3e1147b6d4 100644 --- a/lib/api/multiObjectDelete.js +++ b/lib/api/multiObjectDelete.js @@ -372,7 +372,7 @@ function getObjMetadataAndDelete(authInfo, canonicalID, request, objMD, authInfo, canonicalID, null, request, deleteInfo.newDeleteMarker, null, overheadField, log, 's3:ObjectRemoved:DeleteMarkerCreated', (err, result) => - callback(err, objMD, deleteInfo, result.versionId)); + callback(err, objMD, deleteInfo, result?.versionId)); }, ], (err, objMD, deleteInfo, versionId) => { if (err === skipError) { diff --git a/tests/unit/api/createAndStoreObject.js b/tests/unit/api/createAndStoreObject.js index 1c28cce4f1..e3f8fc187a 100644 --- a/tests/unit/api/createAndStoreObject.js +++ b/tests/unit/api/createAndStoreObject.js @@ -9,6 +9,7 @@ const { storage } = require('arsenal'); const sinon = require('sinon'); const { bucketPut } = require('../../../lib/api/bucketPut'); +const { data } = require('../../../lib/data/wrapper'); const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const metadata = require('../metadataswitch'); const rawCreateAndStoreObject = require('../../../lib/api/apiUtils/object/createAndStoreObject'); @@ -139,6 +140,43 @@ describe('createAndStoreObject', () => { const objMD = await getObjectMDAsync(bucketName, objectKey, {}); assert(objMD.isDeleteMarker); }); + + // The marker must still reach the backend, which uses it to drop the + // live version there. + it('should ignore the request checksum for a delete marker on an external location', async () => { + const putStub = sinon.stub(data.client, 'put').yields(null, 'key', 'versionId'); + const externalBucket = 'test-bucket-external'; + const bucketRequest = new DummyRequest({ + bucketName: externalBucket, + namespace: 'default', + headers: { host: `${externalBucket}.s3.amazonaws.com` }, + url: '/', + post: '' + + '' + + 'awsbackend' + + '', + }); + await promisify(bucketPut)(authInfo, bucketRequest, log); + const bucket = await promisify(metadata.getBucket.bind(metadata))(externalBucket, log); + + const request = new DummyRequest({ + bucketName: externalBucket, + namespace: 'default', + objectKey, + headers: { 'x-amz-checksum-crc32': 'AAAAAQ==' }, + url: `/${externalBucket}/${objectKey}`, + }); + + await createAndStoreObject(externalBucket, bucket, objectKey, null, + authInfo, canonicalID, null, request, true, null, + ['overhead'], log, 's3:ObjectRemoved:DeleteMarkerCreated'); + + assert(putStub.calledOnce, 'delete marker must still reach the backend'); + assert(putStub.firstCall.args[2].isDeleteMarker); + const objMD = await getObjectMDAsync(externalBucket, objectKey, {}); + assert(objMD.isDeleteMarker); + }); }); describe('Archived object replacement', () => {