Skip to content

Commit cd01f9b

Browse files
leif-scalityleif-scality
committed
CLDSRV-887: ListMultipartUploads return checksum algo and type
1 parent f8d623c commit cd01f9b

3 files changed

Lines changed: 305 additions & 18 deletions

File tree

‎lib/api/listMultipartUploads.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ const monitoring = require('../utilities/monitoringHandler');
2424
<IsTruncated>true</IsTruncated>
2525
<Upload>
2626
<Key>my-divisor</Key>
27+
<ChecksumAlgorithm>CRC32</ChecksumAlgorithm>
28+
<ChecksumType>COMPOSITE</ChecksumType>
2729
<UploadId>XMgbGlrZSBlbHZpbmcncyBub3QgaGF2aW5nIG11Y2ggbHVjaw</UploadId>
2830
<Initiator>
2931
<ID>arn:aws:iam::111122223333:user/

‎tests/functional/aws-node-sdk/test/object/listMpuChecksum.js‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
const assert = require('assert');
2+
const {
3+
CreateBucketCommand,
4+
CreateMultipartUploadCommand,
5+
AbortMultipartUploadCommand,
6+
ListMultipartUploadsCommand,
7+
DeleteBucketCommand,
8+
} = require('@aws-sdk/client-s3');
9+
10+
const withV4 = require('../support/withV4');
11+
const BucketUtility = require('../../lib/utility/bucket-util');
12+
13+
const bucket = `list-mpu-checksum-test-${Date.now()}`;
14+
const key = 'test-checksum-key';
15+
16+
describe('ListMultipartUploads checksum fields', () =>
17+
withV4(sigCfg => {
18+
let bucketUtil;
19+
let s3;
20+
21+
before(async () => {
22+
bucketUtil = new BucketUtility('default', sigCfg);
23+
s3 = bucketUtil.s3;
24+
await s3.send(new CreateBucketCommand({ Bucket: bucket }));
25+
});
26+
27+
after(async () => {
28+
await bucketUtil.empty(bucket);
29+
await s3.send(new DeleteBucketCommand({ Bucket: bucket }));
30+
});
31+
32+
describe('MPU created without checksum algorithm', () => {
33+
let uploadId;
34+
35+
before(async () => {
36+
const res = await s3.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: key }));
37+
uploadId = res.UploadId;
38+
});
39+
40+
after(async () => {
41+
await s3.send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: key, UploadId: uploadId }));
42+
});
43+
44+
it('should not include ChecksumAlgorithm or ChecksumType in listed uploads', async () => {
45+
const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket }));
46+
assert.strictEqual(Uploads.length, 1);
47+
assert.strictEqual(Uploads[0].UploadId, uploadId);
48+
assert.strictEqual(Uploads[0].ChecksumAlgorithm, undefined);
49+
assert.strictEqual(Uploads[0].ChecksumType, undefined);
50+
});
51+
});
52+
53+
describe('MPU created with checksum algorithm', () => {
54+
const cases = [
55+
{ algo: 'CRC32', expectedType: 'COMPOSITE' },
56+
{ algo: 'CRC32C', expectedType: 'COMPOSITE' },
57+
{ algo: 'CRC64NVME', expectedType: 'FULL_OBJECT' },
58+
{ algo: 'SHA1', expectedType: 'COMPOSITE' },
59+
{ algo: 'SHA256', expectedType: 'COMPOSITE' },
60+
];
61+
62+
cases.forEach(({ algo, expectedType }) => {
63+
describe(`${algo}`, () => {
64+
let uploadId;
65+
66+
before(async () => {
67+
const res = await s3.send(new CreateMultipartUploadCommand({
68+
Bucket: bucket, Key: key, ChecksumAlgorithm: algo,
69+
}));
70+
uploadId = res.UploadId;
71+
});
72+
73+
after(async () => {
74+
await s3.send(new AbortMultipartUploadCommand({
75+
Bucket: bucket, Key: key, UploadId: uploadId,
76+
}));
77+
});
78+
79+
it(`should return ChecksumAlgorithm ${algo} and ChecksumType ${expectedType}`, async () => {
80+
const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket }));
81+
assert.strictEqual(Uploads.length, 1);
82+
assert.strictEqual(Uploads[0].UploadId, uploadId);
83+
assert.strictEqual(Uploads[0].ChecksumAlgorithm, algo);
84+
assert.strictEqual(Uploads[0].ChecksumType, expectedType);
85+
});
86+
});
87+
});
88+
});
89+
90+
describe('MPU created with explicit algorithm and type', () => {
91+
let uploadId;
92+
93+
before(async () => {
94+
const res = await s3.send(new CreateMultipartUploadCommand({
95+
Bucket: bucket, Key: key, ChecksumAlgorithm: 'CRC32', ChecksumType: 'FULL_OBJECT',
96+
}));
97+
uploadId = res.UploadId;
98+
});
99+
100+
after(async () => {
101+
await s3.send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: key, UploadId: uploadId }));
102+
});
103+
104+
it('should return the explicit ChecksumAlgorithm and ChecksumType', async () => {
105+
const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket }));
106+
assert.strictEqual(Uploads.length, 1);
107+
assert.strictEqual(Uploads[0].UploadId, uploadId);
108+
assert.strictEqual(Uploads[0].ChecksumAlgorithm, 'CRC32');
109+
assert.strictEqual(Uploads[0].ChecksumType, 'FULL_OBJECT');
110+
});
111+
});
112+
113+
describe('multiple MPUs with mixed checksum settings', () => {
114+
const uploads = [];
115+
116+
before(async () => {
117+
const noChecksum = await s3.send(new CreateMultipartUploadCommand({
118+
Bucket: bucket, Key: `${key}-no-checksum`,
119+
}));
120+
uploads.push({ key: `${key}-no-checksum`, uploadId: noChecksum.UploadId });
121+
122+
const withChecksum = await s3.send(new CreateMultipartUploadCommand({
123+
Bucket: bucket, Key: `${key}-with-checksum`, ChecksumAlgorithm: 'SHA256',
124+
}));
125+
uploads.push({ key: `${key}-with-checksum`, uploadId: withChecksum.UploadId });
126+
});
127+
128+
after(async () => {
129+
await Promise.all(uploads.map(u =>
130+
s3.send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: u.key, UploadId: u.uploadId }))
131+
));
132+
});
133+
134+
it('should return checksum fields only for uploads that have them', async () => {
135+
const { Uploads } = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucket }));
136+
assert.strictEqual(Uploads.length, 2);
137+
138+
const noChecksumUpload = Uploads.find(u => u.UploadId === uploads[0].uploadId);
139+
assert.strictEqual(noChecksumUpload.ChecksumAlgorithm, undefined);
140+
assert.strictEqual(noChecksumUpload.ChecksumType, undefined);
141+
142+
const withChecksumUpload = Uploads.find(u => u.UploadId === uploads[1].uploadId);
143+
assert.strictEqual(withChecksumUpload.ChecksumAlgorithm, 'SHA256');
144+
assert.strictEqual(withChecksumUpload.ChecksumType, 'COMPOSITE');
145+
});
146+
});
147+
})
148+
);

‎tests/unit/api/listMultipartUploads.js‎

Lines changed: 155 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,7 @@ const querystring = require('querystring');
44
const { parseString } = require('xml2js');
55

66
const { bucketPut } = require('../../../lib/api/bucketPut');
7-
const initiateMultipartUpload
8-
= require('../../../lib/api/initiateMultipartUpload');
7+
const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload');
98
const listMultipartUploads = require('../../../lib/api/listMultipartUploads');
109
const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers');
1110

@@ -195,8 +194,7 @@ describe('listMultipartUploads API', () => {
195194
});
196195
});
197196

198-
it('should return key following specified ' +
199-
'key-marker', done => {
197+
it('should return key following specified key-marker', done => {
200198
const testListRequest = {
201199
bucketName,
202200
namespace,
@@ -208,22 +206,161 @@ describe('listMultipartUploads API', () => {
208206

209207
async.waterfall([
210208
next => bucketPut(authInfo, testPutBucketRequest, log, next),
211-
(corsHeaders, next) => initiateMultipartUpload(authInfo,
212-
testInitiateMPURequest1, log, next),
213-
(result, corsHeaders, next) => initiateMultipartUpload(authInfo,
214-
testInitiateMPURequest2, log, next),
215-
(result, corsHeaders, next) => initiateMultipartUpload(authInfo,
216-
testInitiateMPURequest3, log, next),
217-
(result, corsHeaders, next) => listMultipartUploads(authInfo,
218-
testListRequest, log, next),
219-
(result, corsHeaders, next) =>
220-
parseString(result, corsHeaders, next),
209+
(corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next),
210+
(result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest2, log, next),
211+
(result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest3, log, next),
212+
(result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next),
213+
(result, corsHeaders, next) => parseString(result, corsHeaders, next),
221214
],
222215
(err, result) => {
223-
assert.strictEqual(result.ListMultipartUploadsResult
224-
.Upload[0].Key[0], objectName2);
225-
assert.strictEqual(result.ListMultipartUploadsResult
226-
.Upload[1], undefined);
216+
assert.strictEqual(result.ListMultipartUploadsResult.Upload[0].Key[0], objectName2);
217+
assert.strictEqual(result.ListMultipartUploadsResult.Upload[1], undefined);
218+
done();
219+
});
220+
});
221+
222+
it('should include ChecksumAlgorithm and ChecksumType when set on MPU', done => {
223+
const checksumKey = 'checksum-object';
224+
const testInitChecksumRequest = {
225+
bucketName,
226+
namespace,
227+
objectKey: checksumKey,
228+
headers: { 'x-amz-checksum-algorithm': 'CRC32' },
229+
url: `/${bucketName}/${checksumKey}?uploads`,
230+
actionImplicitDenies: false,
231+
};
232+
const testListRequest = {
233+
bucketName,
234+
namespace,
235+
headers: { host: '/' },
236+
url: `/${bucketName}?uploads`,
237+
query: {},
238+
actionImplicitDenies: false,
239+
};
240+
241+
async.waterfall([
242+
next => bucketPut(authInfo, testPutBucketRequest, log, next),
243+
(corsHeaders, next) => initiateMultipartUpload(authInfo, testInitChecksumRequest, log, next),
244+
(result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next),
245+
(result, corsHeaders, next) => parseString(result, corsHeaders, next),
246+
],
247+
(err, result) => {
248+
const upload = result.ListMultipartUploadsResult.Upload[0];
249+
assert.strictEqual(upload.Key[0], checksumKey);
250+
assert.strictEqual(upload.ChecksumAlgorithm[0], 'CRC32');
251+
assert.strictEqual(upload.ChecksumType[0], 'COMPOSITE');
252+
done();
253+
});
254+
});
255+
256+
it('should not include ChecksumAlgorithm or ChecksumType when not set on MPU', done => {
257+
const testListRequest = {
258+
bucketName,
259+
namespace,
260+
headers: { host: '/' },
261+
url: `/${bucketName}?uploads`,
262+
query: {},
263+
actionImplicitDenies: false,
264+
};
265+
266+
async.waterfall([
267+
next => bucketPut(authInfo, testPutBucketRequest, log, next),
268+
(corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next),
269+
(result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next),
270+
(result, corsHeaders, next) => parseString(result, corsHeaders, next),
271+
],
272+
(err, result) => {
273+
const upload = result.ListMultipartUploadsResult.Upload[0];
274+
assert.strictEqual(upload.Key[0], objectName1);
275+
assert.strictEqual(upload.ChecksumAlgorithm, undefined);
276+
assert.strictEqual(upload.ChecksumType, undefined);
277+
done();
278+
});
279+
});
280+
281+
it('should include ChecksumAlgorithm and ChecksumType with explicit type', done => {
282+
const checksumKey = 'checksum-explicit';
283+
const testInitChecksumRequest = {
284+
bucketName,
285+
namespace,
286+
objectKey: checksumKey,
287+
headers: {
288+
'x-amz-checksum-algorithm': 'CRC32',
289+
'x-amz-checksum-type': 'FULL_OBJECT',
290+
},
291+
url: `/${bucketName}/${checksumKey}?uploads`,
292+
actionImplicitDenies: false,
293+
};
294+
const testListRequest = {
295+
bucketName,
296+
namespace,
297+
headers: { host: '/' },
298+
url: `/${bucketName}?uploads`,
299+
query: {},
300+
actionImplicitDenies: false,
301+
};
302+
303+
async.waterfall([
304+
next => bucketPut(authInfo, testPutBucketRequest, log, next),
305+
(corsHeaders, next) => initiateMultipartUpload(authInfo, testInitChecksumRequest, log, next),
306+
(result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next),
307+
(result, corsHeaders, next) => parseString(result, corsHeaders, next),
308+
],
309+
(err, result) => {
310+
const upload = result.ListMultipartUploadsResult.Upload[0];
311+
assert.strictEqual(upload.Key[0], checksumKey);
312+
assert.strictEqual(upload.ChecksumAlgorithm[0], 'CRC32');
313+
assert.strictEqual(upload.ChecksumType[0], 'FULL_OBJECT');
314+
done();
315+
});
316+
});
317+
318+
it('should list mixed uploads with and without checksum correctly', done => {
319+
const checksumKey = 'aaa-checksum-object';
320+
const noChecksumKey = 'zzz-no-checksum-object';
321+
const testInitChecksumRequest = {
322+
bucketName,
323+
namespace,
324+
objectKey: checksumKey,
325+
headers: { 'x-amz-checksum-algorithm': 'SHA256' },
326+
url: `/${bucketName}/${checksumKey}?uploads`,
327+
actionImplicitDenies: false,
328+
};
329+
const testInitNoChecksumRequest = {
330+
bucketName,
331+
namespace,
332+
objectKey: noChecksumKey,
333+
headers: {},
334+
url: `/${bucketName}/${noChecksumKey}?uploads`,
335+
actionImplicitDenies: false,
336+
};
337+
const testListRequest = {
338+
bucketName,
339+
namespace,
340+
headers: { host: '/' },
341+
url: `/${bucketName}?uploads`,
342+
query: {},
343+
actionImplicitDenies: false,
344+
};
345+
346+
async.waterfall([
347+
next => bucketPut(authInfo, testPutBucketRequest, log, next),
348+
(corsHeaders, next) => initiateMultipartUpload(authInfo, testInitChecksumRequest, log, next),
349+
(result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitNoChecksumRequest, log, next),
350+
(result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next),
351+
(result, corsHeaders, next) => parseString(result, corsHeaders, next),
352+
],
353+
(err, result) => {
354+
const uploads = result.ListMultipartUploadsResult.Upload;
355+
assert.strictEqual(uploads.length, 2);
356+
357+
const withChecksum = uploads.find(u => u.Key[0] === checksumKey);
358+
assert.strictEqual(withChecksum.ChecksumAlgorithm[0], 'SHA256');
359+
assert.strictEqual(withChecksum.ChecksumType[0], 'COMPOSITE');
360+
361+
const withoutChecksum = uploads.find(u => u.Key[0] === noChecksumKey);
362+
assert.strictEqual(withoutChecksum.ChecksumAlgorithm, undefined);
363+
assert.strictEqual(withoutChecksum.ChecksumType, undefined);
227364
done();
228365
});
229366
});

0 commit comments

Comments
 (0)