diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py index 1089d6fefb..1f937ded06 100644 --- a/swift/common/middleware/slo.py +++ b/swift/common/middleware/slo.py @@ -258,6 +258,10 @@ class StaticLargeObject(object): problem_segments.append([quote(obj_path), 'Size Mismatch']) if seg_dict['etag'] != head_seg_resp.etag: problem_segments.append([quote(obj_path), 'Etag Mismatch']) + if 'X-Static-Large-Object' in head_seg_resp.headers or \ + 'X-Object-Manifest' in head_seg_resp.headers: + problem_segments.append( + [quote(obj_path), 'Segments cannot be Large Objects']) if head_seg_resp.last_modified: last_modified = head_seg_resp.last_modified else: diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 9d5cc1cb42..4917a14028 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -173,6 +173,9 @@ class SegmentedIterable(object): '%(path)s etag: %(r_etag)s != %(s_etag)s.' % {'path': path, 'r_etag': resp.etag, 's_etag': self.segment_dict['hash']})) + if 'X-Static-Large-Object' in resp.headers: + raise SloSegmentError(_( + 'SLO can not be made of other SLOs: %s' % path)) self.segment_iter = resp.app_iter # See NOTE: swift_conn at top of file about this. self.segment_iter_swift_conn = getattr(resp, 'swift_conn', None) diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 9dd14b29cf..9d3577276d 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -273,7 +273,7 @@ def fake_http_connect(*code_iter, **kwargs): class FakeConn(object): def __init__(self, status, etag=None, body='', timestamp='1', - expect_status=None): + expect_status=None, headers=None): self.status = status if expect_status is None: self.expect_status = self.status @@ -286,6 +286,7 @@ def fake_http_connect(*code_iter, **kwargs): self.received = 0 self.etag = etag self.body = body + self.headers = headers or {} self.timestamp = timestamp def getresponse(self): @@ -329,8 +330,7 @@ def fake_http_connect(*code_iter, **kwargs): pass if 'slow' in kwargs: headers['content-length'] = '4' - if 'headers' in kwargs: - headers.update(kwargs['headers']) + headers.update(self.headers) return headers.items() def read(self, amt=None): @@ -354,6 +354,11 @@ def fake_http_connect(*code_iter, **kwargs): timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter)) etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) + if isinstance(kwargs.get('headers'), list): + headers_iter = iter(kwargs['headers']) + else: + headers_iter = iter([kwargs.get('headers', {})] * len(code_iter)) + x = kwargs.get('missing_container', [False] * len(code_iter)) if not isinstance(x, (tuple, list)): x = [x] * len(code_iter) @@ -378,6 +383,7 @@ def fake_http_connect(*code_iter, **kwargs): else: expect_status = status etag = etag_iter.next() + headers = headers_iter.next() timestamp = timestamps_iter.next() if status <= 0: @@ -387,6 +393,6 @@ def fake_http_connect(*code_iter, **kwargs): else: body = body_iter.next() return FakeConn(status, etag, body=body, timestamp=timestamp, - expect_status=expect_status) + expect_status=expect_status, headers=headers) return connect diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py index 069539b308..0bf4b6748e 100644 --- a/test/unit/common/middleware/test_slo.py +++ b/test/unit/common/middleware/test_slo.py @@ -36,10 +36,11 @@ class FakeApp(object): cont_len = 100 if obj == 'small_object': cont_len = 10 - return Response( - status=200, - headers={'etag': 'etagoftheobjectsegment', - 'Content-Length': cont_len})(env, start_response) + headers = {'etag': 'etagoftheobjectsegment', + 'Content-Length': cont_len} + if obj == 'slob': + headers['X-Static-Large-Object'] = 'true' + return Response(status=200, headers=headers)(env, start_response) if env['PATH_INFO'].startswith('/test_good_check/'): j, v, a, cont, obj = env['PATH_INFO'].split('/') etag, size = obj.split('_') @@ -307,7 +308,8 @@ class TestStaticLargeObject(unittest.TestCase): bad_data = json.dumps( [{'path': '/c/a_1', 'etag': 'a', 'size_bytes': '1'}, {'path': '/c/a_2', 'etag': 'a', 'size_bytes': '1'}, - {'path': '/d/b_2', 'etag': 'b', 'size_bytes': '2'}]) + {'path': '/d/b_2', 'etag': 'b', 'size_bytes': '2'}, + {'path': '/d/slob', 'etag': 'b', 'size_bytes': '2'}]) req = Request.blank( '/test_good/A/c/man?multipart-manifest=put', environ={'REQUEST_METHOD': 'PUT'}, @@ -316,14 +318,17 @@ class TestStaticLargeObject(unittest.TestCase): try: self.slo.handle_multipart_put(req) except HTTPException, e: - self.assertEquals(self.app.calls, 3) + self.assertEquals(self.app.calls, 4) data = json.loads(e.body) errors = data['Errors'] self.assertEquals(errors[0][0], '/test_good/A/c/a_1') self.assertEquals(errors[0][1], 'Size Mismatch') self.assertEquals(errors[2][1], '400 Bad Request') - self.assertEquals(errors[-1][0], '/test_good/A/d/b_2') - self.assertEquals(errors[-1][1], 'Etag Mismatch') + self.assertEquals(errors[4][0], '/test_good/A/d/b_2') + self.assertEquals(errors[4][1], 'Etag Mismatch') + self.assertEquals(errors[-1][0], '/test_good/A/d/slob') + self.assertEquals(errors[-1][1], + 'Segments cannot be Large Objects') else: self.assert_(False) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 36cd1ef0e7..dde7ebb64d 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -1021,8 +1021,8 @@ class TestObjectController(unittest.TestCase): 200, # GET listing1 200, # GET seg01 200, # GET seg02 - headers={"X-Static-Large-Object": "True", - 'content-type': 'text/html; swift_bytes=4'}, + headers=[{}, {}, {"X-Static-Large-Object": "True", + 'content-type': 'text/html; swift_bytes=4'}, {}, {}], body_iter=response_bodies, give_connect=capture_requested_paths) @@ -1176,8 +1176,64 @@ class TestObjectController(unittest.TestCase): 200, # GET listing1 200, # GET seg01 200, # GET seg02 - headers={"X-Static-Large-Object": "True", - 'content-type': 'text/html; swift_bytes=4'}, + headers=[{}, {}, {"X-Static-Large-Object": "True", + 'content-type': 'text/html; swift_bytes=4'}, {}, {}], + body_iter=response_bodies, + give_connect=capture_requested_paths) + req = Request.blank('/a/c/manifest') + resp = controller.GET(req) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.body, 'Aa') # dropped connection + self.assertEqual(resp.content_length, 4) # content incomplete + self.assertEqual(resp.content_type, 'text/html') + + self.assertEqual( + requested, + [['HEAD', '/a', {}], + ['HEAD', '/a/c', {}], + ['GET', '/a/c/manifest', {}], + ['GET', '/a/d1/seg01', {}], + ['GET', '/a/d2/seg02', {}]]) + + def test_GET_nested_slo(self): + listing = [{"hash": "98568d540134639be4655198a36614a4", + "last_modified": "2012-11-08T04:05:37.866820", + "bytes": 2, + "name": "/d1/seg01", + "content_type": "application/octet-stream"}, + {"hash": "d526f1c8ef6c1e4e980e2b8471352d23", + "last_modified": "2012-11-08T04:05:37.846710", + "bytes": 2, + "name": "/d2/seg02", + "content_type": "application/octet-stream"}] + + response_bodies = ( + '', # HEAD /a + '', # HEAD /a/c + simplejson.dumps(listing), # GET manifest + 'Aa', # GET seg01 + 'Bb') # GET seg02 + with save_globals(): + controller = proxy_server.ObjectController( + self.app, 'a', 'c', 'manifest') + + requested = [] + + def capture_requested_paths(ipaddr, port, device, partition, + method, path, headers=None, + query_string=None): + qs_dict = dict(urlparse.parse_qsl(query_string or '')) + requested.append([method, path, qs_dict]) + + slob_headers = {"X-Static-Large-Object": "True", + 'content-type': 'text/html; swift_bytes=4'} + set_http_connect( + 200, # HEAD /a + 200, # HEAD /a/c + 200, # GET listing1 + 200, # GET seg01 + 200, # GET seg02 + headers=[{}, {}, slob_headers, {}, slob_headers], body_iter=response_bodies, give_connect=capture_requested_paths) req = Request.blank('/a/c/manifest') @@ -1236,8 +1292,8 @@ class TestObjectController(unittest.TestCase): 200, # GET listing1 200, # GET seg01 404, # GET seg02 - headers={"X-Static-Large-Object": "True", - 'content-type': 'text/html; swift_bytes=4'}, + headers=[{}, {}, {"X-Static-Large-Object": "True", + 'content-type': 'text/html; swift_bytes=4'}, {}, {}], body_iter=response_bodies, give_connect=capture_requested_paths) req = Request.blank('/a/c/manifest')

AltStyle によって変換されたページ (->オリジナル) /