Provide a method for retrieving on-disk metadata

Code Issues Proposed changes

We hide the internal dictionary for the metadata providing a method to
retrieve it to abstract away the implementation details of how
DiskFile object provides and maintains that metadata.
This is in anticipation of the DiskFile API refactoring.
Change-Id: I1c0dc01a4680bd435512405e2d31fba24421720a
Signed-off-by: Peter Portante <peter.portante@redhat.com>

This commit is contained in:

Peter Portante

2013年09月03日 11:10:22 -04:00

parent 8a255a3415

commit 698023f477

3 changed files with 82 additions and 64 deletions

Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL

Download Patch File Download Diff File Expand all files Collapse all files

36
swift/obj/diskfile.py

View File

@@ -380,7 +380,7 @@ class DiskFile(object):

self.device_path = join(path, device)

self.tmpdir = join(path, device, 'tmp')

self.logger = logger

self.metadata = {}

self._metadata = {}

self.data_file = None

self.fp = None

self.iter_etag = None

@@ -464,8 +464,8 @@ class DiskFile(object):

pull the metadata from the tombstone file which has the timestamp.

"""

with open(ts_file) as fp:

self.metadata = read_metadata(fp)

self.metadata['deleted'] = True

self._metadata = read_metadata(fp)

self._metadata['deleted'] = True

def _verify_name(self):

"""

@@ -473,7 +473,7 @@ class DiskFile(object):

named.

"""

try:

mname = self.metadata['name']

mname = self._metadata['name']

except KeyError:

pass

else:

@@ -495,13 +495,13 @@ class DiskFile(object):

datafile_metadata = read_metadata(fp)

if meta_file:

with open(meta_file) as mfp:

self.metadata = read_metadata(mfp)

self._metadata = read_metadata(mfp)

sys_metadata = dict(

[(key, val) for key, val in datafile_metadata.iteritems()

if key.lower() in DATAFILE_SYSTEM_META])

self.metadata.update(sys_metadata)

self._metadata.update(sys_metadata)

else:

self.metadata = datafile_metadata

self._metadata = datafile_metadata

self._verify_name()

self.data_file = data_file

return fp

@@ -586,8 +586,8 @@ class DiskFile(object):

return

if self.iter_etag and self.started_at_0 and self.read_to_eof and \

'ETag' in self.metadata and \

self.iter_etag.hexdigest() != self.metadata.get('ETag'):

'ETag' in self._metadata and \

self.iter_etag.hexdigest() != self._metadata.get('ETag'):

self.quarantine()

def close(self, verify_file=True):

@@ -611,6 +611,14 @@ class DiskFile(object):

self.fp.close()

self.fp = None

def get_metadata(self):

"""

Provide the metadata for an object as a dictionary.

:returns: object's metadata dictionary

"""

return self._metadata

def is_deleted(self):

"""

Check if the file is deleted.

@@ -618,7 +626,7 @@ class DiskFile(object):

:returns: True if the file doesn't exist or has been flagged as

deleted.

"""

return not self.data_file or 'deleted' in self.metadata

return not self.data_file or 'deleted' in self._metadata

def is_expired(self):

"""

@@ -626,8 +634,8 @@ class DiskFile(object):

:returns: True if the file has an X-Delete-At in the past

"""

return ('X-Delete-At' in self.metadata and

int(self.metadata['X-Delete-At']) <= time.time())

return ('X-Delete-At' in self._metadata and

int(self._metadata['X-Delete-At']) <= time.time())

@contextmanager

def create(self, size=None):

@@ -713,8 +721,8 @@ class DiskFile(object):

if self.data_file:

file_size = self.threadpool.run_in_thread(

getsize, self.data_file)

if 'Content-Length' in self.metadata:

metadata_size = int(self.metadata['Content-Length'])

if 'Content-Length' in self._metadata:

metadata_size = int(self._metadata['Content-Length'])

if file_size != metadata_size:

raise DiskFileError(

'Content-Length of %s does not match file size '

65
swift/obj/server.py

View File

@@ -305,7 +305,8 @@ class ObjectController(object):

except (DiskFileError, DiskFileNotExist):

disk_file.quarantine()

return HTTPNotFound(request=request)

orig_timestamp = disk_file.metadata.get('X-Timestamp', '0')

orig_metadata = disk_file.get_metadata()

orig_timestamp = orig_metadata.get('X-Timestamp', '0')

if orig_timestamp >= request.headers['x-timestamp']:

return HTTPConflict(request=request)

metadata = {'X-Timestamp': request.headers['x-timestamp']}

@@ -315,7 +316,7 @@ class ObjectController(object):

if header_key in request.headers:

header_caps = header_key.title()

metadata[header_caps] = request.headers[header_key]

old_delete_at = int(disk_file.metadata.get('X-Delete-At') or 0)

old_delete_at = int(orig_metadata.get('X-Delete-At') or 0)

if old_delete_at != new_delete_at:

if new_delete_at:

self.delete_at_update('PUT', new_delete_at, account, container,

@@ -354,8 +355,9 @@ class ObjectController(object):

obj)

except DiskFileDeviceUnavailable:

return HTTPInsufficientStorage(drive=device, request=request)

old_delete_at = int(disk_file.metadata.get('X-Delete-At') or 0)

orig_timestamp = disk_file.metadata.get('X-Timestamp')

orig_metadata = disk_file.get_metadata()

old_delete_at = int(orig_metadata.get('X-Delete-At') or 0)

orig_timestamp = orig_metadata.get('X-Timestamp')

if orig_timestamp and orig_timestamp >= request.headers['x-timestamp']:

return HTTPConflict(request=request)

upload_expiration = time.time() + self.max_upload_time

@@ -414,10 +416,10 @@ class ObjectController(object):

self.container_update(

'PUT', account, container, obj, request,

HeaderKeyDict({

'x-size': disk_file.metadata['Content-Length'],

'x-content-type': disk_file.metadata['Content-Type'],

'x-timestamp': disk_file.metadata['X-Timestamp'],

'x-etag': disk_file.metadata['ETag']}),

'x-size': metadata['Content-Length'],

'x-content-type': metadata['Content-Type'],

'x-timestamp': metadata['X-Timestamp'],

'x-etag': metadata['ETag']}),

device)

resp = HTTPCreated(request=request, etag=etag)

return resp

@@ -443,14 +445,15 @@ class ObjectController(object):

except (DiskFileError, DiskFileNotExist):

disk_file.quarantine()

return HTTPNotFound(request=request)

metadata = disk_file.get_metadata()

if request.headers.get('if-match') not in (None, '*') and \

disk_file.metadata['ETag'] not in request.if_match:

metadata['ETag'] not in request.if_match:

disk_file.close()

return HTTPPreconditionFailed(request=request)

if request.headers.get('if-none-match') is not None:

if disk_file.metadata['ETag'] in request.if_none_match:

if metadata['ETag'] in request.if_none_match:

resp = HTTPNotModified(request=request)

resp.etag = disk_file.metadata['ETag']

resp.etag = metadata['ETag']

disk_file.close()

return resp

try:

@@ -460,7 +463,7 @@ class ObjectController(object):

return HTTPPreconditionFailed(request=request)

if if_unmodified_since and \

datetime.fromtimestamp(

float(disk_file.metadata['X-Timestamp']), UTC) > \

float(metadata['X-Timestamp']), UTC) > \

if_unmodified_since:

disk_file.close()

return HTTPPreconditionFailed(request=request)

@@ -471,29 +474,29 @@ class ObjectController(object):

return HTTPPreconditionFailed(request=request)

if if_modified_since and \

datetime.fromtimestamp(

float(disk_file.metadata['X-Timestamp']), UTC) < \

float(metadata['X-Timestamp']), UTC) < \

if_modified_since:

disk_file.close()

return HTTPNotModified(request=request)

response = Response(app_iter=disk_file,

request=request, conditional_response=True)

response.headers['Content-Type'] = disk_file.metadata.get(

response.headers['Content-Type'] = metadata.get(

'Content-Type', 'application/octet-stream')

for key, value in disk_file.metadata.iteritems():

for key, value in metadata.iteritems():

if key.lower().startswith('x-object-meta-') or \

key.lower() in self.allowed_headers:

response.headers[key] = value

response.etag = disk_file.metadata['ETag']

response.last_modified = float(disk_file.metadata['X-Timestamp'])

response.etag = metadata['ETag']

response.last_modified = float(metadata['X-Timestamp'])

response.content_length = file_size

if response.content_length < self.keep_cache_size and \

(self.keep_cache_private or

('X-Auth-Token' not in request.headers and

'X-Storage-Token' not in request.headers)):

disk_file.keep_cache = True

if 'Content-Encoding' in disk_file.metadata:

response.content_encoding = disk_file.metadata['Content-Encoding']

response.headers['X-Timestamp'] = disk_file.metadata['X-Timestamp']

if 'Content-Encoding' in metadata:

response.content_encoding = metadata['Content-Encoding']

response.headers['X-Timestamp'] = metadata['X-Timestamp']

return request.get_response(response)

@public

@@ -515,19 +518,20 @@ class ObjectController(object):

disk_file.quarantine()

return HTTPNotFound(request=request)

response = Response(request=request, conditional_response=True)

response.headers['Content-Type'] = disk_file.metadata.get(

metadata = disk_file.get_metadata()

response.headers['Content-Type'] = metadata.get(

'Content-Type', 'application/octet-stream')

for key, value in disk_file.metadata.iteritems():

for key, value in metadata.iteritems():

if key.lower().startswith('x-object-meta-') or \

key.lower() in self.allowed_headers:

response.headers[key] = value

response.etag = disk_file.metadata['ETag']

response.last_modified = float(disk_file.metadata['X-Timestamp'])

response.etag = metadata['ETag']

response.last_modified = float(metadata['X-Timestamp'])

# Needed for container sync feature

response.headers['X-Timestamp'] = disk_file.metadata['X-Timestamp']

response.headers['X-Timestamp'] = metadata['X-Timestamp']

response.content_length = file_size

if 'Content-Encoding' in disk_file.metadata:

response.content_encoding = disk_file.metadata['Content-Encoding']

if 'Content-Encoding' in metadata:

response.content_encoding = metadata['Content-Encoding']

return response

@public

@@ -545,17 +549,18 @@ class ObjectController(object):

obj)

except DiskFileDeviceUnavailable:

return HTTPInsufficientStorage(drive=device, request=request)

orig_metadata = disk_file.get_metadata()

if 'x-if-delete-at' in request.headers and \

int(request.headers['x-if-delete-at']) != \

int(disk_file.metadata.get('X-Delete-At') or 0):

int(orig_metadata.get('X-Delete-At') or 0):

return HTTPPreconditionFailed(

request=request,

body='X-If-Delete-At and X-Delete-At do not match')

old_delete_at = int(disk_file.metadata.get('X-Delete-At') or 0)

old_delete_at = int(orig_metadata.get('X-Delete-At') or 0)

if old_delete_at:

self.delete_at_update('DELETE', old_delete_at, account,

container, obj, request, device)

orig_timestamp = disk_file.metadata.get('X-Timestamp', 0)

orig_timestamp = orig_metadata.get('X-Timestamp', 0)

req_timestamp = request.headers['X-Timestamp']

if disk_file.is_deleted() or disk_file.is_expired():

response_class = HTTPNotFound

45
test/unit/obj/test_diskfile.py

View File

@@ -387,12 +387,17 @@ class TestDiskFile(unittest.TestCase):

FakeLogger(), keep_data_fp=keep_data_fp)

return df

def test_get_metadata(self):

df = self._create_test_file('1234567890', timestamp=42)

md = df.get_metadata()

self.assertEquals(md['X-Timestamp'], normalize_timestamp(42))

def test_disk_file_default_disallowed_metadata(self):

# build an object with some meta (ts 41)

orig_metadata = {'X-Object-Meta-Key1': 'Value1',

'Content-Type': 'text/garbage'}

df = self._get_disk_file(ts=41, extra_metadata=orig_metadata)

self.assertEquals('1024', df.metadata['Content-Length'])

self.assertEquals('1024', df._metadata['Content-Length'])

# write some new metadata (fast POST, don't send orig meta, ts 42)

df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',

FakeLogger())

@@ -400,11 +405,11 @@ class TestDiskFile(unittest.TestCase):

df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',

FakeLogger())

# non-fast-post updateable keys are preserved

self.assertEquals('text/garbage', df.metadata['Content-Type'])

self.assertEquals('text/garbage', df._metadata['Content-Type'])

# original fast-post updateable keys are removed

self.assert_('X-Object-Meta-Key1' not in df.metadata)

self.assert_('X-Object-Meta-Key1' not in df._metadata)

# new fast-post updateable keys are added

self.assertEquals('Value2', df.metadata['X-Object-Meta-Key2'])

self.assertEquals('Value2', df._metadata['X-Object-Meta-Key2'])

def test_disk_file_app_iter_corners(self):

df = self._create_test_file('1234567890')

@@ -721,10 +726,10 @@ class TestDiskFile(unittest.TestCase):

self._create_ondisk_file(df, 'A', ext='.data', timestamp=5)

df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',

FakeLogger())

self.assertTrue('X-Timestamp' in df.metadata)

self.assertEquals(df.metadata['X-Timestamp'], normalize_timestamp(10))

self.assertTrue('deleted' in df.metadata)

self.assertTrue(df.metadata['deleted'])

self.assertTrue('X-Timestamp' in df._metadata)

self.assertEquals(df._metadata['X-Timestamp'], normalize_timestamp(10))

self.assertTrue('deleted' in df._metadata)

self.assertTrue(df._metadata['deleted'])

def test_ondisk_search_loop_meta_ts_data(self):

df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',

@@ -737,9 +742,9 @@ class TestDiskFile(unittest.TestCase):

self._create_ondisk_file(df, 'A', ext='.data', timestamp=5)

df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',

FakeLogger())

self.assertTrue('X-Timestamp' in df.metadata)

self.assertEquals(df.metadata['X-Timestamp'], normalize_timestamp(8))

self.assertTrue('deleted' in df.metadata)

self.assertTrue('X-Timestamp' in df._metadata)

self.assertEquals(df._metadata['X-Timestamp'], normalize_timestamp(8))

self.assertTrue('deleted' in df._metadata)

def test_ondisk_search_loop_meta_data_ts(self):

df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',

@@ -752,9 +757,9 @@ class TestDiskFile(unittest.TestCase):

self._create_ondisk_file(df, '', ext='.ts', timestamp=5)

df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',

FakeLogger())

self.assertTrue('X-Timestamp' in df.metadata)

self.assertEquals(df.metadata['X-Timestamp'], normalize_timestamp(10))

self.assertTrue('deleted' not in df.metadata)

self.assertTrue('X-Timestamp' in df._metadata)

self.assertEquals(df._metadata['X-Timestamp'], normalize_timestamp(10))

self.assertTrue('deleted' not in df._metadata)

def test_ondisk_search_loop_data_meta_ts(self):

df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',

@@ -767,9 +772,9 @@ class TestDiskFile(unittest.TestCase):

self._create_ondisk_file(df, '', ext='.meta', timestamp=5)

df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',

FakeLogger())

self.assertTrue('X-Timestamp' in df.metadata)

self.assertEquals(df.metadata['X-Timestamp'], normalize_timestamp(10))

self.assertTrue('deleted' not in df.metadata)

self.assertTrue('X-Timestamp' in df._metadata)

self.assertEquals(df._metadata['X-Timestamp'], normalize_timestamp(10))

self.assertTrue('deleted' not in df._metadata)

def test_ondisk_search_loop_wayward_files_ignored(self):

df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',

@@ -783,9 +788,9 @@ class TestDiskFile(unittest.TestCase):

self._create_ondisk_file(df, '', ext='.meta', timestamp=5)

df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',

FakeLogger())

self.assertTrue('X-Timestamp' in df.metadata)

self.assertEquals(df.metadata['X-Timestamp'], normalize_timestamp(10))

self.assertTrue('deleted' not in df.metadata)

self.assertTrue('X-Timestamp' in df._metadata)

self.assertEquals(df._metadata['X-Timestamp'], normalize_timestamp(10))

self.assertTrue('deleted' not in df._metadata)

def test_ondisk_search_loop_listdir_error(self):

df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o',