s3api: Add support for crc64nvme checksum calculation

Code Issues Proposed changes

Add anycrc as a soft dependency in case ISA-L isn't available.
Plus we'll want it later: when we start writing down checksums,
we'll need it to combine per-part checksums for MPUs.
Like with crc32c, we won't provide any pure-python version as the
CPU-intensiveness could present a DoS vector. Worst case, we 501
as before.
Co-Authored-By: Tim Burke <tim.burke@gmail.com>
Signed-off-by: Tim Burke <tim.burke@gmail.com>
Change-Id: Ia05e5677a8ca89a62b142078abfb7371b1badd3f
Signed-off-by: Alistair Coles <alistairncoles@gmail.com>

This commit is contained in:

Alistair Coles

2025年04月02日 10:24:04 +01:00

parent be56c1e258

commit 404e1f2732

6 changed files with 239 additions and 34 deletions

Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL

Download Patch File Download Diff File Expand all files Collapse all files

24
test/s3api/test_object_checksums.py

View File

@@ -191,7 +191,11 @@ class ObjectChecksumMixin(object):

'ChecksumAlgorithm': self.ALGORITHM,

}

if boto_at_least(1, 36):

checksum_kwargs['ChecksumType'] = 'COMPOSITE'

if self.ALGORITHM == 'CRC64NVME':

# crc64nvme only allows full-object

checksum_kwargs['ChecksumType'] = 'FULL_OBJECT'

else:

checksum_kwargs['ChecksumType'] = 'COMPOSITE'

obj_name = self.create_name(self.ALGORITHM + '-mpu-complete-good')

create_mpu_resp = self.client.create_multipart_upload(

@@ -247,6 +251,24 @@ class TestObjectChecksumCRC32C(ObjectChecksumMixin, BaseS3TestCaseWithBucket):

super().setUpClass()

class TestObjectChecksumCRC64NVME(ObjectChecksumMixin,

BaseS3TestCaseWithBucket):

ALGORITHM = 'CRC64NVME'

EXPECTED = 'rosUhgp5mIg='

INVALID = 'rosUhgp5mIh='

BAD = 'sosUhgp5mIg='

@classmethod

def setUpClass(cls):

if [int(x) for x in botocore.__version__.split('.')] < [1, 36]:

raise SkipTest('botocore cannot crc64nvme (run '

'`pip install -U boto3 botocore`)')

if not botocore.httpchecksum.HAS_CRT:

raise SkipTest('botocore cannot crc64nvme (run '

'`pip install awscrt`)')

super().setUpClass()

class TestObjectChecksumSHA1(ObjectChecksumMixin, BaseS3TestCaseWithBucket):

ALGORITHM = 'SHA1'

EXPECTED = '98O8HYCOBHMq32eZZczDTKeuNEE='

s3api: Add support for crc64nvme checksum calculation

24 test/s3api/test_object_checksums.py Unescape Escape View File

24
test/s3api/test_object_checksums.py

View File