Hide the file descriptor and disk write methodology for PUTs

Code Issues Proposed changes

Towards moving the DiskFile class into place as the API definition for
pluggable DiskFile backends, we hide the file descriptor and the
method of writing data to disks. The mkstemp() method has been renamed
to writer(), and no longer returns an fd, but a new object that
encapsulates the state tracked for writes. This new object is then
used directly to perform the reminder of the write operations and
application of required semantics.
Change-Id: Ib37ed37b34a2ce6b442d69f83ca011c918114434
Signed-off-by: Peter Portante <peter.portante@redhat.com>

This commit is contained in:

Peter Portante

2013年04月18日 20:42:36 -04:00

committed by Gerrit Code Review

parent 6b4cba8371

commit d0a27f477b

3 changed files with 190 additions and 166 deletions

Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL

Download Patch File Download Diff File Expand all files Collapse all files

110
test/unit/obj/test_auditor.py

View File

@@ -62,17 +62,17 @@ class TestAuditor(unittest.TestCase):

self.auditor = auditor.AuditorWorker(self.conf, self.logger)

data = '0' * 1024

etag = md5()

with self.disk_file.mkstemp() as fd:

os.write(fd, data)

with self.disk_file.writer() as writer:

writer.write(data)

etag.update(data)

etag = etag.hexdigest()

timestamp = str(normalize_timestamp(time.time()))

metadata = {

'ETag': etag,

'X-Timestamp': timestamp,

'Content-Length': str(os.fstat(fd).st_size),

'Content-Length': str(os.fstat(writer.fd).st_size),

}

self.disk_file.put(fd, 1024, metadata)

writer.put(metadata)

pre_quarantines = self.auditor.quarantines

self.auditor.object_audit(

@@ -80,7 +80,7 @@ class TestAuditor(unittest.TestCase):

'sda', '0')

self.assertEquals(self.auditor.quarantines, pre_quarantines)

os.write(fd, 'extra_data')

os.write(writer.fd, 'extra_data')

self.auditor.object_audit(

os.path.join(self.disk_file.datadir, timestamp + '.data'),

'sda', '0')

@@ -91,35 +91,39 @@ class TestAuditor(unittest.TestCase):

data = '0' * 1024

etag = md5()

timestamp = str(normalize_timestamp(time.time()))

with self.disk_file.mkstemp() as fd:

os.write(fd, data)

with self.disk_file.writer() as writer:

writer.write(data)

etag.update(data)

etag = etag.hexdigest()

metadata = {

'ETag': etag,

'X-Timestamp': timestamp,

'Content-Length': str(os.fstat(fd).st_size),

'Content-Length': str(os.fstat(writer.fd).st_size),

}

self.disk_file.put(fd, 1024, metadata)

writer.put(metadata)

pre_quarantines = self.auditor.quarantines

# remake so it will have metadata

self.disk_file = DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o',

self.logger)

self.auditor.object_audit(

os.path.join(self.disk_file.datadir, timestamp + '.data'),

'sda', '0')

self.assertEquals(self.auditor.quarantines, pre_quarantines)

etag = md5()

etag.update('1' + '0' * 1023)

etag = etag.hexdigest()

metadata['ETag'] = etag

write_metadata(fd, metadata)

# remake so it will have metadata

self.disk_file = DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o',

self.logger)

self.auditor.object_audit(

os.path.join(self.disk_file.datadir, timestamp + '.data'),

'sda', '0')

self.assertEquals(self.auditor.quarantines, pre_quarantines + 1)

self.auditor.object_audit(

os.path.join(self.disk_file.datadir, timestamp + '.data'),

'sda', '0')

self.assertEquals(self.auditor.quarantines, pre_quarantines)

etag = md5()

etag.update('1' + '0' * 1023)

etag = etag.hexdigest()

metadata['ETag'] = etag

with self.disk_file.writer() as writer:

writer.write(data)

writer.put(metadata)

self.auditor.object_audit(

os.path.join(self.disk_file.datadir, timestamp + '.data'),

'sda', '0')

self.assertEquals(self.auditor.quarantines, pre_quarantines + 1)

def test_object_audit_no_meta(self):

timestamp = str(normalize_timestamp(time.time()))

@@ -152,17 +156,16 @@ class TestAuditor(unittest.TestCase):

pre_quarantines = self.auditor.quarantines

data = '0' * 1024

etag = md5()

with self.disk_file.mkstemp() as fd:

os.write(fd, data)

with self.disk_file.writer() as writer:

writer.write(data)

etag.update(data)

etag = etag.hexdigest()

metadata = {

'ETag': etag,

'X-Timestamp': timestamp,

'Content-Length': str(os.fstat(fd).st_size),

'Content-Length': str(os.fstat(writer.fd).st_size),

}

self.disk_file.put(fd, 1024, metadata)

self.disk_file.close()

writer.put(metadata)

self.auditor.audit_all_objects()

self.assertEquals(self.auditor.quarantines, pre_quarantines)

@@ -172,18 +175,17 @@ class TestAuditor(unittest.TestCase):

pre_quarantines = self.auditor.quarantines

data = '0' * 1024

etag = md5()

with self.disk_file.mkstemp() as fd:

os.write(fd, data)

with self.disk_file.writer() as writer:

writer.write(data)

etag.update(data)

etag = etag.hexdigest()

metadata = {

'ETag': etag,

'X-Timestamp': timestamp,

'Content-Length': str(os.fstat(fd).st_size),

'Content-Length': str(os.fstat(writer.fd).st_size),

}

self.disk_file.put(fd, 1024, metadata)

self.disk_file.close()

os.write(fd, 'extra_data')

writer.put(metadata)

os.write(writer.fd, 'extra_data')

self.auditor.audit_all_objects()

self.assertEquals(self.auditor.quarantines, pre_quarantines + 1)

@@ -193,34 +195,32 @@ class TestAuditor(unittest.TestCase):

pre_quarantines = self.auditor.quarantines

data = '0' * 10

etag = md5()

with self.disk_file.mkstemp() as fd:

os.write(fd, data)

with self.disk_file.writer() as writer:

writer.write(data)

etag.update(data)

etag = etag.hexdigest()

metadata = {

'ETag': etag,

'X-Timestamp': timestamp,

'Content-Length': str(os.fstat(fd).st_size),

'Content-Length': str(os.fstat(writer.fd).st_size),

}

self.disk_file.put(fd, 10, metadata)

self.disk_file.close()

writer.put(metadata)

self.auditor.audit_all_objects()

self.disk_file = DiskFile(self.devices, 'sdb', '0', 'a', 'c',

'ob', self.logger)

data = '1' * 10

etag = md5()

with self.disk_file.mkstemp() as fd:

os.write(fd, data)

with self.disk_file.writer() as writer:

writer.write(data)

etag.update(data)

etag = etag.hexdigest()

metadata = {

'ETag': etag,

'X-Timestamp': timestamp,

'Content-Length': str(os.fstat(fd).st_size),

'Content-Length': str(os.fstat(writer.fd).st_size),

}

self.disk_file.put(fd, 10, metadata)

self.disk_file.close()

os.write(fd, 'extra_data')

writer.put(metadata)

os.write(writer.fd, 'extra_data')

self.auditor.audit_all_objects()

self.assertEquals(self.auditor.quarantines, pre_quarantines + 1)

@@ -229,21 +229,21 @@ class TestAuditor(unittest.TestCase):

self.auditor.log_time = 0

data = '0' * 1024

etag = md5()

with self.disk_file.mkstemp() as fd:

os.write(fd, data)

with self.disk_file.writer() as writer:

writer.write(data)

etag.update(data)

etag = etag.hexdigest()

metadata = {

'ETag': etag,

'X-Timestamp': str(normalize_timestamp(time.time())),

'Content-Length': str(os.fstat(fd).st_size),

'Content-Length': str(os.fstat(writer.fd).st_size),

}

self.disk_file.put(fd, 1024, metadata)

writer.put(metadata)

etag = md5()

etag.update('1' + '0' * 1023)

etag = etag.hexdigest()

metadata['ETag'] = etag

write_metadata(fd, metadata)

write_metadata(writer.fd, metadata)

quarantine_path = os.path.join(self.devices,

'sda', 'quarantined', 'objects')

@@ -268,18 +268,18 @@ class TestAuditor(unittest.TestCase):

fp.close()

etag = md5()

with self.disk_file.mkstemp() as fd:

with self.disk_file.writer() as writer:

etag = etag.hexdigest()

metadata = {

'ETag': etag,

'X-Timestamp': str(normalize_timestamp(time.time())),

'Content-Length': 10,

}

self.disk_file.put(fd, 10, metadata)

writer.put(metadata)

etag = md5()

etag = etag.hexdigest()

metadata['ETag'] = etag

write_metadata(fd, metadata)

write_metadata(writer.fd, metadata)

if self.disk_file.data_file:

return self.disk_file.data_file

return ts_file_path

60
test/unit/obj/test_server.py

View File

@@ -145,7 +145,7 @@ class TestDiskFile(unittest.TestCase):

tmpdir = os.path.join(self.testdir, 'sda1', 'tmp')

os.rmdir(tmpdir)

with object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c',

'o', FakeLogger()).mkstemp():

'o', FakeLogger()).writer() as writer:

self.assert_(os.path.exists(tmpdir))

def test_iter_hook(self):

@@ -153,7 +153,7 @@ class TestDiskFile(unittest.TestCase):

def hook():

hook_call_count[0] += 1

df = self._get_data_file(fsize=65, csize=8, iter_hook=hook)

df = self._get_disk_file(fsize=65, csize=8, iter_hook=hook)

for _ in df:

pass

@@ -204,7 +204,7 @@ class TestDiskFile(unittest.TestCase):

self.assert_(os.path.isdir(double_uuid_path))

self.assert_('-' in os.path.basename(double_uuid_path))

def _get_data_file(self, invalid_type=None, obj_name='o',

def _get_disk_file(self, invalid_type=None, obj_name='o',

fsize=1024, csize=8, extension='.data', ts=None,

iter_hook=None):

'''returns a DiskFile'''

@@ -216,25 +216,25 @@ class TestDiskFile(unittest.TestCase):

timestamp = ts

else:

timestamp = str(normalize_timestamp(time()))

with df.mkstemp() as fd:

os.write(fd, data)

with df.writer() as writer:

writer.write(data)

etag.update(data)

etag = etag.hexdigest()

metadata = {

'ETag': etag,

'X-Timestamp': timestamp,

'Content-Length': str(os.fstat(fd).st_size),

'Content-Length': str(os.fstat(writer.fd).st_size),

}

df.put(fd, fsize, metadata, extension=extension)

writer.put(metadata, extension=extension)

if invalid_type == 'ETag':

etag = md5()

etag.update('1' + '0' * (fsize - 1))

etag = etag.hexdigest()

metadata['ETag'] = etag

object_server.write_metadata(fd, metadata)

object_server.write_metadata(writer.fd, metadata)

if invalid_type == 'Content-Length':

metadata['Content-Length'] = fsize - 1

object_server.write_metadata(fd, metadata)

object_server.write_metadata(writer.fd, metadata)

df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c',

obj_name, FakeLogger(),

@@ -248,43 +248,43 @@ class TestDiskFile(unittest.TestCase):

return df

def test_quarantine_valids(self):

df = self._get_data_file(obj_name='1')

df = self._get_disk_file(obj_name='1')

for chunk in df:

pass

self.assertFalse(df.quarantined_dir)

df = self._get_data_file(obj_name='2', csize=1)

df = self._get_disk_file(obj_name='2', csize=1)

for chunk in df:

pass

self.assertFalse(df.quarantined_dir)

df = self._get_data_file(obj_name='3', csize=100000)

df = self._get_disk_file(obj_name='3', csize=100000)

for chunk in df:

pass

self.assertFalse(df.quarantined_dir)

def run_quarantine_invalids(self, invalid_type):

df = self._get_data_file(invalid_type=invalid_type, obj_name='1')

df = self._get_disk_file(invalid_type=invalid_type, obj_name='1')

for chunk in df:

pass

self.assertTrue(df.quarantined_dir)

df = self._get_data_file(invalid_type=invalid_type,

df = self._get_disk_file(invalid_type=invalid_type,

obj_name='2', csize=1)

for chunk in df:

pass

self.assertTrue(df.quarantined_dir)

df = self._get_data_file(invalid_type=invalid_type,

df = self._get_disk_file(invalid_type=invalid_type,

obj_name='3', csize=100000)

for chunk in df:

pass

self.assertTrue(df.quarantined_dir)

df = self._get_data_file(invalid_type=invalid_type, obj_name='4')

df = self._get_disk_file(invalid_type=invalid_type, obj_name='4')

self.assertFalse(df.quarantined_dir)

df = self._get_data_file(invalid_type=invalid_type, obj_name='5')

df = self._get_disk_file(invalid_type=invalid_type, obj_name='5')

for chunk in df.app_iter_range(0, df.unit_test_len):

pass

self.assertTrue(df.quarantined_dir)

df = self._get_data_file(invalid_type=invalid_type, obj_name='6')

df = self._get_disk_file(invalid_type=invalid_type, obj_name='6')

for chunk in df.app_iter_range(0, df.unit_test_len + 100):

pass

self.assertTrue(df.quarantined_dir)

@@ -293,15 +293,15 @@ class TestDiskFile(unittest.TestCase):

# in a quarantine, even if the whole file isn't check-summed

if invalid_type in ('Zero-Byte', 'Content-Length'):

expected_quar = True

df = self._get_data_file(invalid_type=invalid_type, obj_name='7')

df = self._get_disk_file(invalid_type=invalid_type, obj_name='7')

for chunk in df.app_iter_range(1, df.unit_test_len):

pass

self.assertEquals(bool(df.quarantined_dir), expected_quar)

df = self._get_data_file(invalid_type=invalid_type, obj_name='8')

df = self._get_disk_file(invalid_type=invalid_type, obj_name='8')

for chunk in df.app_iter_range(0, df.unit_test_len - 1):

pass

self.assertEquals(bool(df.quarantined_dir), expected_quar)

df = self._get_data_file(invalid_type=invalid_type, obj_name='8')

df = self._get_disk_file(invalid_type=invalid_type, obj_name='8')

for chunk in df.app_iter_range(1, df.unit_test_len + 1):

pass

self.assertEquals(bool(df.quarantined_dir), expected_quar)

@@ -312,20 +312,20 @@ class TestDiskFile(unittest.TestCase):

self.run_quarantine_invalids('Zero-Byte')

def test_quarantine_deleted_files(self):

df = self._get_data_file(invalid_type='Content-Length',

df = self._get_disk_file(invalid_type='Content-Length',

extension='.data')

df.close()

self.assertTrue(df.quarantined_dir)

df = self._get_data_file(invalid_type='Content-Length',

df = self._get_disk_file(invalid_type='Content-Length',

extension='.ts')

df.close()

self.assertFalse(df.quarantined_dir)

df = self._get_data_file(invalid_type='Content-Length',

df = self._get_disk_file(invalid_type='Content-Length',

extension='.ts')

self.assertRaises(DiskFileNotExist, df.get_data_file_size)

def test_put_metadata(self):

df = self._get_data_file()

df = self._get_disk_file()

ts = time()

metadata = { 'X-Timestamp': ts, 'X-Object-Meta-test': 'data' }

df.put_metadata(metadata)

@@ -335,7 +335,7 @@ class TestDiskFile(unittest.TestCase):

self.assertTrue(exp_name in set(dl))

def test_put_metadata_ts(self):

df = self._get_data_file()

df = self._get_disk_file()

ts = time()

metadata = { 'X-Timestamp': ts, 'X-Object-Meta-test': 'data' }

df.put_metadata(metadata, tombstone=True)

@@ -345,9 +345,9 @@ class TestDiskFile(unittest.TestCase):

self.assertTrue(exp_name in set(dl))

def test_unlinkold(self):

df1 = self._get_data_file()

df1 = self._get_disk_file()

future_time = str(normalize_timestamp(time() + 100))

df2 = self._get_data_file(ts=future_time)

df2 = self._get_disk_file(ts=future_time)

self.assertEquals(len(os.listdir(df1.datadir)), 2)

df1.unlinkold(future_time)

self.assertEquals(len(os.listdir(df1.datadir)), 1)

@@ -358,7 +358,7 @@ class TestDiskFile(unittest.TestCase):

def err():

raise Exception("bad")

df = self._get_data_file(fsize=1024 * 1024 * 2)

df = self._get_disk_file(fsize=1024 * 1024 * 2)

df._handle_close_quarantine = err

for chunk in df:

pass

@@ -367,7 +367,7 @@ class TestDiskFile(unittest.TestCase):

self.assertEquals(len(df.logger.log_dict['error']), 1)

def test_quarantine_twice(self):

df = self._get_data_file(invalid_type='Content-Length',

df = self._get_disk_file(invalid_type='Content-Length',

extension='.data')

self.assert_(os.path.isfile(df.data_file))

quar_dir = df.quarantine()