slobjects can not be made of other slobjects

Code Issues Proposed changes

Change-Id: Ib500f5fc2e890c3d23641350815a910168bcd543

This commit is contained in:

David Goetz

2013年04月11日 12:52:33 -07:00

parent c496ba1878

commit 3f21e14398

5 changed files with 92 additions and 18 deletions

Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL

Download Patch File Download Diff File Expand all files Collapse all files

4
swift/common/middleware/slo.py

View File

@@ -258,6 +258,10 @@ class StaticLargeObject(object):

problem_segments.append([quote(obj_path), 'Size Mismatch'])

if seg_dict['etag'] != head_seg_resp.etag:

problem_segments.append([quote(obj_path), 'Etag Mismatch'])

if 'X-Static-Large-Object' in head_seg_resp.headers or \

'X-Object-Manifest' in head_seg_resp.headers:

problem_segments.append(

[quote(obj_path), 'Segments cannot be Large Objects'])

if head_seg_resp.last_modified:

last_modified = head_seg_resp.last_modified

else:

3
swift/proxy/controllers/obj.py

View File

@@ -173,6 +173,9 @@ class SegmentedIterable(object):

'%(path)s etag: %(r_etag)s != %(s_etag)s.' %

{'path': path, 'r_etag': resp.etag,

's_etag': self.segment_dict['hash']}))

if 'X-Static-Large-Object' in resp.headers:

raise SloSegmentError(_(

'SLO can not be made of other SLOs: %s' % path))

self.segment_iter = resp.app_iter

# See NOTE: swift_conn at top of file about this.

self.segment_iter_swift_conn = getattr(resp, 'swift_conn', None)

14
test/unit/init.py

View File

@@ -273,7 +273,7 @@ def fake_http_connect(*code_iter, **kwargs):

class FakeConn(object):

def __init__(self, status, etag=None, body='', timestamp='1',

expect_status=None):

expect_status=None, headers=None):

self.status = status

if expect_status is None:

self.expect_status = self.status

@@ -286,6 +286,7 @@ def fake_http_connect(*code_iter, **kwargs):

self.received = 0

self.etag = etag

self.body = body

self.headers = headers or {}

self.timestamp = timestamp

def getresponse(self):

@@ -329,8 +330,7 @@ def fake_http_connect(*code_iter, **kwargs):

pass

if 'slow' in kwargs:

headers['content-length'] = '4'

if 'headers' in kwargs:

headers.update(kwargs['headers'])

headers.update(self.headers)

return headers.items()

def read(self, amt=None):

@@ -354,6 +354,11 @@ def fake_http_connect(*code_iter, **kwargs):

timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter))

etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter))

if isinstance(kwargs.get('headers'), list):

headers_iter = iter(kwargs['headers'])

else:

headers_iter = iter([kwargs.get('headers', {})] * len(code_iter))

x = kwargs.get('missing_container', [False] * len(code_iter))

if not isinstance(x, (tuple, list)):

x = [x] * len(code_iter)

@@ -378,6 +383,7 @@ def fake_http_connect(*code_iter, **kwargs):

else:

expect_status = status

etag = etag_iter.next()

headers = headers_iter.next()

timestamp = timestamps_iter.next()

if status <= 0:

@@ -387,6 +393,6 @@ def fake_http_connect(*code_iter, **kwargs):

else:

body = body_iter.next()

return FakeConn(status, etag, body=body, timestamp=timestamp,

expect_status=expect_status)

expect_status=expect_status, headers=headers)

return connect

21
test/unit/common/middleware/test_slo.py

View File

@@ -36,10 +36,11 @@ class FakeApp(object):

cont_len = 100

if obj == 'small_object':

cont_len = 10

return Response(

status=200,

headers={'etag': 'etagoftheobjectsegment',

'Content-Length': cont_len})(env, start_response)

headers = {'etag': 'etagoftheobjectsegment',

'Content-Length': cont_len}

if obj == 'slob':

headers['X-Static-Large-Object'] = 'true'

return Response(status=200, headers=headers)(env, start_response)

if env['PATH_INFO'].startswith('/test_good_check/'):

j, v, a, cont, obj = env['PATH_INFO'].split('/')

etag, size = obj.split('_')

@@ -307,7 +308,8 @@ class TestStaticLargeObject(unittest.TestCase):

bad_data = json.dumps(

[{'path': '/c/a_1', 'etag': 'a', 'size_bytes': '1'},

{'path': '/c/a_2', 'etag': 'a', 'size_bytes': '1'},

{'path': '/d/b_2', 'etag': 'b', 'size_bytes': '2'}])

{'path': '/d/b_2', 'etag': 'b', 'size_bytes': '2'},

{'path': '/d/slob', 'etag': 'b', 'size_bytes': '2'}])

req = Request.blank(

'/test_good/A/c/man?multipart-manifest=put',

environ={'REQUEST_METHOD': 'PUT'},

@@ -316,14 +318,17 @@ class TestStaticLargeObject(unittest.TestCase):

try:

self.slo.handle_multipart_put(req)

except HTTPException, e:

self.assertEquals(self.app.calls, 3)

self.assertEquals(self.app.calls, 4)

data = json.loads(e.body)

errors = data['Errors']

self.assertEquals(errors[0][0], '/test_good/A/c/a_1')

self.assertEquals(errors[0][1], 'Size Mismatch')

self.assertEquals(errors[2][1], '400 Bad Request')

self.assertEquals(errors[-1][0], '/test_good/A/d/b_2')

self.assertEquals(errors[-1][1], 'Etag Mismatch')

self.assertEquals(errors[4][0], '/test_good/A/d/b_2')

self.assertEquals(errors[4][1], 'Etag Mismatch')

self.assertEquals(errors[-1][0], '/test_good/A/d/slob')

self.assertEquals(errors[-1][1],

'Segments cannot be Large Objects')

else:

self.assert_(False)

68
test/unit/proxy/test_server.py

View File

@@ -1021,8 +1021,8 @@ class TestObjectController(unittest.TestCase):

200, # GET listing1

200, # GET seg01

200, # GET seg02

headers={"X-Static-Large-Object": "True",

'content-type': 'text/html; swift_bytes=4'},

headers=[{}, {}, {"X-Static-Large-Object": "True",

'content-type': 'text/html; swift_bytes=4'}, {}, {}],

body_iter=response_bodies,

give_connect=capture_requested_paths)

@@ -1176,8 +1176,64 @@ class TestObjectController(unittest.TestCase):

200, # GET listing1

200, # GET seg01

200, # GET seg02

headers={"X-Static-Large-Object": "True",

'content-type': 'text/html; swift_bytes=4'},

headers=[{}, {}, {"X-Static-Large-Object": "True",

'content-type': 'text/html; swift_bytes=4'}, {}, {}],

body_iter=response_bodies,

give_connect=capture_requested_paths)

req = Request.blank('/a/c/manifest')

resp = controller.GET(req)

self.assertEqual(resp.status_int, 200)

self.assertEqual(resp.body, 'Aa') # dropped connection

self.assertEqual(resp.content_length, 4) # content incomplete

self.assertEqual(resp.content_type, 'text/html')

self.assertEqual(

requested,

[['HEAD', '/a', {}],

['HEAD', '/a/c', {}],

['GET', '/a/c/manifest', {}],

['GET', '/a/d1/seg01', {}],

['GET', '/a/d2/seg02', {}]])

def test_GET_nested_slo(self):

listing = [{"hash": "98568d540134639be4655198a36614a4",

"last_modified": "2012年11月08日T04:05:37.866820",

"bytes": 2,

"name": "/d1/seg01",

"content_type": "application/octet-stream"},

{"hash": "d526f1c8ef6c1e4e980e2b8471352d23",

"last_modified": "2012年11月08日T04:05:37.846710",

"bytes": 2,

"name": "/d2/seg02",

"content_type": "application/octet-stream"}]

response_bodies = (

'', # HEAD /a

'', # HEAD /a/c

simplejson.dumps(listing), # GET manifest

'Aa', # GET seg01

'Bb') # GET seg02

with save_globals():

controller = proxy_server.ObjectController(

self.app, 'a', 'c', 'manifest')

requested = []

def capture_requested_paths(ipaddr, port, device, partition,

method, path, headers=None,

query_string=None):

qs_dict = dict(urlparse.parse_qsl(query_string or ''))

requested.append([method, path, qs_dict])

slob_headers = {"X-Static-Large-Object": "True",

'content-type': 'text/html; swift_bytes=4'}

set_http_connect(

200, # HEAD /a

200, # HEAD /a/c

200, # GET listing1

200, # GET seg01

200, # GET seg02

headers=[{}, {}, slob_headers, {}, slob_headers],

body_iter=response_bodies,

give_connect=capture_requested_paths)

req = Request.blank('/a/c/manifest')

@@ -1236,8 +1292,8 @@ class TestObjectController(unittest.TestCase):

200, # GET listing1

200, # GET seg01

404, # GET seg02

headers={"X-Static-Large-Object": "True",

'content-type': 'text/html; swift_bytes=4'},

headers=[{}, {}, {"X-Static-Large-Object": "True",

'content-type': 'text/html; swift_bytes=4'}, {}, {}],

body_iter=response_bodies,

give_connect=capture_requested_paths)

req = Request.blank('/a/c/manifest')