diff --git a/doc/source/logs.rst b/doc/source/logs.rst index e2cd553dc4..2d7ed423b9 100644 --- a/doc/source/logs.rst +++ b/doc/source/logs.rst @@ -141,6 +141,7 @@ SYM :ref:`symlink` SH :ref:`sharding_doc` S3 :ref:`s3api` OV :ref:`object_versioning` +EQ :ref:`etag_quoter` ======================= ============================= diff --git a/doc/source/middleware.rst b/doc/source/middleware.rst index ca48bda952..ef282bfa35 100644 --- a/doc/source/middleware.rst +++ b/doc/source/middleware.rst @@ -207,6 +207,15 @@ Encryption middleware should be deployed in conjunction with the :members: :show-inheritance: +.. _etag_quoter: + +Etag Quoter +=========== + +.. automodule:: swift.common.middleware.etag_quoter + :members: + :show-inheritance: + .. _formpost: FormPost diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index c1468047f9..ea979dbfb7 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -850,6 +850,17 @@ use = egg:swift#name_check # maximum_length = 255 # forbidden_regexp = /\./|/\.\./|/\.$|/\.\.$ +# Note: Etag quoter should be placed just after cache in the pipeline. +[filter:etag-quoter] +use = egg:swift#etag_quoter +# Historically, Swift has emitted bare MD5 hex digests as ETags, which is not +# RFC compliant. With this middleware in the pipeline, users can opt-in to +# RFC-compliant ETags on a per-account or per-container basis. +# +# Set to true to enable RFC-compliant ETags cluster-wide by default. Users +# can still opt-out by setting appropriate account or container metadata. +# enable_by_default = false + [filter:list-endpoints] use = egg:swift#list_endpoints # list_endpoints_path = /endpoints/ diff --git a/setup.cfg b/setup.cfg index 84d3ba7be4..4c84177525 100644 --- a/setup.cfg +++ b/setup.cfg @@ -126,6 +126,7 @@ paste.filter_factory = symlink = swift.common.middleware.symlink:filter_factory s3api = swift.common.middleware.s3api.s3api:filter_factory s3token = swift.common.middleware.s3api.s3token:filter_factory + etag_quoter = swift.common.middleware.etag_quoter:filter_factory swift.diskfile = replication.fs = swift.obj.diskfile:DiskFileManager diff --git a/swift/common/middleware/etag_quoter.py b/swift/common/middleware/etag_quoter.py new file mode 100644 index 0000000000..8cc527579b --- /dev/null +++ b/swift/common/middleware/etag_quoter.py @@ -0,0 +1,127 @@ +# Copyright (c) 2010-2020 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This middleware fix the Etag header of responses so that it is RFC compliant. +`RFC 7232 `__ specifies that +the value of the Etag header must be double quoted. + +It must be placed at the beggining of the pipeline, right after cache:: + + [pipeline:main] + pipeline = ... cache etag-quoter ... + + [filter:etag-quoter] + use = egg:swift#etag_quoter + +Set ``X-Account-Rfc-Compliant-Etags: true`` at the account +level to have any Etags in object responses be double quoted, as in +``"d41d8cd98f00b204e9800998ecf8427e"``. Alternatively, you may +only fix Etags in a single container by setting +``X-Container-Rfc-Compliant-Etags: true`` on the container. +This may be necessary for Swift to work properly with some CDNs. + +Either option may also be explicitly *disabled*, so you may enable quoted +Etags account-wide as above but turn them off for individual containers +with ``X-Container-Rfc-Compliant-Etags: false``. This may be +useful if some subset of applications expect Etags to be bare MD5s. +""" + +from swift.common.constraints import valid_api_version +from swift.common.http import is_success +from swift.common.swob import Request +from swift.common.utils import config_true_value, register_swift_info +from swift.proxy.controllers.base import get_account_info, get_container_info + + +class EtagQuoterMiddleware(object): + def __init__(self, app, conf): + self.app = app + self.conf = conf + + def __call__(self, env, start_response): + req = Request(env) + try: + version, account, container, obj = req.split_path( + 2, 4, rest_with_last=True) + is_swifty_request = valid_api_version(version) + except ValueError: + is_swifty_request = False + + if not is_swifty_request: + return self.app(env, start_response) + + if not obj: + typ = 'Container' if container else 'Account' + client_header = 'X-%s-Rfc-Compliant-Etags' % typ + sysmeta_header = 'X-%s-Sysmeta-Rfc-Compliant-Etags' % typ + if client_header in req.headers: + if req.headers[client_header]: + req.headers[sysmeta_header] = config_true_value( + req.headers[client_header]) + else: + req.headers[sysmeta_header] = '' + if req.headers.get(client_header.replace('X-', 'X-Remove-', 1)): + req.headers[sysmeta_header] = '' + + def translating_start_response(status, headers, exc_info=None): + return start_response(status, [ + (client_header if h.title() == sysmeta_header else h, + v) for h, v in headers + ], exc_info) + + return self.app(env, translating_start_response) + + container_info = get_container_info(env, self.app, 'EQ') + if not container_info or not is_success(container_info['status']): + return self.app(env, start_response) + + flag = container_info.get('sysmeta', {}).get('rfc-compliant-etags') + if flag is None: + account_info = get_account_info(env, self.app, 'EQ') + if not account_info or not is_success(account_info['status']): + return self.app(env, start_response) + + flag = account_info.get('sysmeta', {}).get( + 'rfc-compliant-etags') + + if flag is None: + flag = self.conf.get('enable_by_default', 'false') + + if not config_true_value(flag): + return self.app(env, start_response) + + status, headers, resp_iter = req.call_application(self.app) + + for i, (header, value) in enumerate(headers): + if header.lower() == 'etag': + if not value.startswith(('"', 'W/"')) or \ + not value.endswith('"'): + headers[i] = (header, '"%s"' % value) + + start_response(status, headers) + return resp_iter + + +def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + register_swift_info( + 'etag_quoter', enable_by_default=config_true_value( + conf.get('enable_by_default', 'false'))) + + def etag_quoter_filter(app): + return EtagQuoterMiddleware(app, conf) + return etag_quoter_filter diff --git a/test/functional/test_object.py b/test/functional/test_object.py index f78823a390..ce9730b375 100644 --- a/test/functional/test_object.py +++ b/test/functional/test_object.py @@ -16,6 +16,7 @@ # limitations under the License. import datetime +import hashlib import json import unittest from uuid import uuid4 @@ -1719,6 +1720,58 @@ class TestObject(unittest.TestCase): self.assertEqual(len(final_status[0].childNodes), 1) self.assertEqual(final_status[0].childNodes[0].data, '200 OK') + def test_etag_quoter(self): + if tf.skip: + raise SkipTest + if 'etag_quoter' not in tf.cluster_info: + raise SkipTest("etag-quoter middleware is not enabled") + + def do_head(expect_quoted=False): + def head(url, token, parsed, conn): + conn.request('HEAD', '%s/%s/%s' % ( + parsed.path, self.container, self.obj), '', + {'X-Auth-Token': token}) + return check_response(conn) + + resp = retry(head) + resp.read() + self.assertEqual(resp.status, 200) + expected_etag = hashlib.md5(b'test').hexdigest() + if expect_quoted: + expected_etag = '"%s"' % expected_etag + self.assertEqual(resp.headers['etag'], expected_etag) + + def _post(enable_flag, container_path): + def post(url, token, parsed, conn): + if container_path: + path = '%s/%s' % (parsed.path, self.container) + hdr = 'X-Container-Rfc-Compliant-Etags' + else: + path = parsed.path + hdr = 'X-Account-Rfc-Compliant-Etags' + headers = {hdr: enable_flag, 'X-Auth-Token': token} + conn.request('POST', path, '', headers) + return check_response(conn) + + resp = retry(post) + resp.read() + self.assertEqual(resp.status, 204) + + def post_account(enable_flag): + return _post(enable_flag, False) + + def post_container(enable_flag): + return _post(enable_flag, True) + + do_head() + post_container('t') + do_head(expect_quoted=True) + post_account('t') + post_container('') + do_head(expect_quoted=True) + post_container('f') + do_head() + if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/middleware/test_etag_quoter.py b/test/unit/common/middleware/test_etag_quoter.py new file mode 100644 index 0000000000..3a894c18e7 --- /dev/null +++ b/test/unit/common/middleware/test_etag_quoter.py @@ -0,0 +1,215 @@ +# Copyright (c) 2010-2020 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import unittest + +from swift.common import swob +from swift.common.middleware import etag_quoter +from swift.proxy.controllers.base import get_cache_key + +from test.unit.common.middleware.helpers import FakeSwift + + +def set_info_cache(req, cache_data, account, container=None): + req.environ.setdefault('swift.infocache', {})[ + get_cache_key(account, container)] = cache_data + + +class TestEtagQuoter(unittest.TestCase): + def get_mw(self, conf, etag='unquoted-etag', path=None): + if path is None: + path = '/v1/AUTH_acc/con/some/path/to/obj' + app = FakeSwift() + hdrs = {} if etag is None else {'ETag': etag} + app.register('GET', path, swob.HTTPOk, hdrs) + return etag_quoter.filter_factory({}, **conf)(app) + + @mock.patch('swift.common.middleware.etag_quoter.register_swift_info') + def test_swift_info(self, mock_register): + self.get_mw({}) + self.assertEqual(mock_register.mock_calls, [ + mock.call('etag_quoter', enable_by_default=False)]) + mock_register.reset_mock() + + self.get_mw({'enable_by_default': '1'}) + self.assertEqual(mock_register.mock_calls, [ + mock.call('etag_quoter', enable_by_default=True)]) + mock_register.reset_mock() + + self.get_mw({'enable_by_default': 'no'}) + self.assertEqual(mock_register.mock_calls, [ + mock.call('etag_quoter', enable_by_default=False)]) + + def test_account_on_overrides_cluster_off(self): + req = swob.Request.blank('/v1/AUTH_acc/con/some/path/to/obj') + set_info_cache(req, { + 'status': 200, + 'sysmeta': {'rfc-compliant-etags': '1'}, + }, 'AUTH_acc') + set_info_cache(req, { + 'status': 200, + 'sysmeta': {}, + }, 'AUTH_acc', 'con') + resp = req.get_response(self.get_mw({'enable_by_default': 'false'})) + self.assertEqual(resp.headers['ETag'], '"unquoted-etag"') + + def test_account_off_overrides_cluster_on(self): + req = swob.Request.blank('/v1/AUTH_acc/con/some/path/to/obj') + set_info_cache(req, { + 'status': 200, + 'sysmeta': {'rfc-compliant-etags': 'no'}, + }, 'AUTH_acc') + set_info_cache(req, { + 'status': 200, + 'sysmeta': {}, + }, 'AUTH_acc', 'con') + resp = req.get_response(self.get_mw({'enable_by_default': 'yes'})) + self.assertEqual(resp.headers['ETag'], 'unquoted-etag') + + def test_container_on_overrides_cluster_off(self): + req = swob.Request.blank('/v1/AUTH_acc/con/some/path/to/obj') + set_info_cache(req, { + 'status': 200, + 'sysmeta': {}, + }, 'AUTH_acc') + set_info_cache(req, { + 'status': 200, + 'sysmeta': {'rfc-compliant-etags': 't'}, + }, 'AUTH_acc', 'con') + resp = req.get_response(self.get_mw({'enable_by_default': 'false'})) + self.assertEqual(resp.headers['ETag'], '"unquoted-etag"') + + def test_container_off_overrides_cluster_on(self): + req = swob.Request.blank('/v1/AUTH_acc/con/some/path/to/obj') + set_info_cache(req, { + 'status': 200, + 'sysmeta': {}, + }, 'AUTH_acc') + set_info_cache(req, { + 'status': 200, + 'sysmeta': {'rfc-compliant-etags': '0'}, + }, 'AUTH_acc', 'con') + resp = req.get_response(self.get_mw({'enable_by_default': 'yes'})) + self.assertEqual(resp.headers['ETag'], 'unquoted-etag') + + def test_container_on_overrides_account_off(self): + req = swob.Request.blank('/v1/AUTH_acc/con/some/path/to/obj') + set_info_cache(req, { + 'status': 200, + 'sysmeta': {'rfc-compliant-etags': 'no'}, + }, 'AUTH_acc') + set_info_cache(req, { + 'status': 200, + 'sysmeta': {'rfc-compliant-etags': 't'}, + }, 'AUTH_acc', 'con') + resp = req.get_response(self.get_mw({})) + self.assertEqual(resp.headers['ETag'], '"unquoted-etag"') + + def test_container_off_overrides_account_on(self): + req = swob.Request.blank('/v1/AUTH_acc/con/some/path/to/obj') + set_info_cache(req, { + 'status': 200, + 'sysmeta': {'rfc-compliant-etags': 'yes'}, + }, 'AUTH_acc') + set_info_cache(req, { + 'status': 200, + 'sysmeta': {'rfc-compliant-etags': 'false'}, + }, 'AUTH_acc', 'con') + resp = req.get_response(self.get_mw({})) + self.assertEqual(resp.headers['ETag'], 'unquoted-etag') + + def test_cluster_wide(self): + req = swob.Request.blank('/v1/AUTH_acc/con/some/path/to/obj') + set_info_cache(req, {'status': 200, 'sysmeta': {}}, 'AUTH_acc') + set_info_cache(req, {'status': 200, 'sysmeta': {}}, 'AUTH_acc', 'con') + resp = req.get_response(self.get_mw({'enable_by_default': 't'})) + self.assertEqual(resp.headers['ETag'], '"unquoted-etag"') + + def test_already_valid(self): + req = swob.Request.blank('/v1/AUTH_acc/con/some/path/to/obj') + set_info_cache(req, {'status': 200, 'sysmeta': {}}, 'AUTH_acc') + set_info_cache(req, {'status': 200, 'sysmeta': {}}, 'AUTH_acc', 'con') + resp = req.get_response(self.get_mw({'enable_by_default': 't'}, + '"quoted-etag"')) + self.assertEqual(resp.headers['ETag'], '"quoted-etag"') + + def test_already_weak_but_valid(self): + req = swob.Request.blank('/v1/AUTH_acc/con/some/path/to/obj') + set_info_cache(req, {'status': 200, 'sysmeta': {}}, 'AUTH_acc') + set_info_cache(req, {'status': 200, 'sysmeta': {}}, 'AUTH_acc', 'con') + resp = req.get_response(self.get_mw({'enable_by_default': 't'}, + 'W/"weak-etag"')) + self.assertEqual(resp.headers['ETag'], 'W/"weak-etag"') + + def test_only_half_valid(self): + req = swob.Request.blank('/v1/AUTH_acc/con/some/path/to/obj') + set_info_cache(req, {'status': 200, 'sysmeta': {}}, 'AUTH_acc') + set_info_cache(req, {'status': 200, 'sysmeta': {}}, 'AUTH_acc', 'con') + resp = req.get_response(self.get_mw({'enable_by_default': 't'}, + '"weird-etag')) + self.assertEqual(resp.headers['ETag'], '""weird-etag"') + + def test_no_etag(self): + req = swob.Request.blank('/v1/AUTH_acc/con/some/path/to/obj') + set_info_cache(req, {'status': 200, 'sysmeta': {}}, 'AUTH_acc') + set_info_cache(req, {'status': 200, 'sysmeta': {}}, 'AUTH_acc', 'con') + resp = req.get_response(self.get_mw({'enable_by_default': 't'}, + etag=None)) + self.assertNotIn('ETag', resp.headers) + + def test_non_swift_path(self): + path = '/some/other/location/entirely' + req = swob.Request.blank(path) + resp = req.get_response(self.get_mw({'enable_by_default': 't'}, + path=path)) + self.assertEqual(resp.headers['ETag'], 'unquoted-etag') + + def test_non_object_request(self): + path = '/v1/AUTH_acc/con' + req = swob.Request.blank(path) + resp = req.get_response(self.get_mw({'enable_by_default': 't'}, + path=path)) + self.assertEqual(resp.headers['ETag'], 'unquoted-etag') + + def test_no_container_info(self): + mw = self.get_mw({'enable_by_default': 't'}) + req = swob.Request.blank('/v1/AUTH_acc/con/some/path/to/obj') + set_info_cache(req, {'status': 200, 'sysmeta': {}}, 'AUTH_acc') + mw.app.register('HEAD', '/v1/AUTH_acc/con', + swob.HTTPServiceUnavailable, {}) + resp = req.get_response(mw) + self.assertEqual(resp.headers['ETag'], 'unquoted-etag') + set_info_cache(req, {'status': 404, 'sysmeta': {}}, 'AUTH_acc', 'con') + resp = req.get_response(mw) + self.assertEqual(resp.headers['ETag'], 'unquoted-etag') + set_info_cache(req, {'status': 200, 'sysmeta': {}}, 'AUTH_acc', 'con') + resp = req.get_response(mw) + self.assertEqual(resp.headers['ETag'], '"unquoted-etag"') + + def test_no_account_info(self): + mw = self.get_mw({'enable_by_default': 't'}) + req = swob.Request.blank('/v1/AUTH_acc/con/some/path/to/obj') + mw.app.register('HEAD', '/v1/AUTH_acc', + swob.HTTPServiceUnavailable, {}) + set_info_cache(req, {'status': 200, 'sysmeta': {}}, 'AUTH_acc', 'con') + resp = req.get_response(mw) + self.assertEqual(resp.headers['ETag'], 'unquoted-etag') + set_info_cache(req, {'status': 404, 'sysmeta': {}}, 'AUTH_acc') + resp = req.get_response(mw) + self.assertEqual(resp.headers['ETag'], 'unquoted-etag') + set_info_cache(req, {'status': 200, 'sysmeta': {}}, 'AUTH_acc') + resp = req.get_response(mw) + self.assertEqual(resp.headers['ETag'], '"unquoted-etag"')

AltStyle によって変換されたページ (->オリジナル) /