proxy-logging: Add real-time transfer bytes counters

Code Issues Proposed changes

Currently we can get one proxy-logging transfer stat emission over the
duration of the upload/download. We want another stat coming out of
proxy-logging: something that gets emitted periodically as bytes are
actually sent/received so we can get reasonably accurate point-in-time
breakdowns of bandwidth usage.
Co-Authored-By: Alistair Coles <alistairncoles@gmail.com>
Co-Authored-By: Shreeya Deshpande <shreeyad@nvidia.com>
Change-Id: Ideecd0aa58ddf091c9f25f15022a9066088f532b
Signed-off-by: Yan Xiao <yanxiao@nvidia.com>

This commit is contained in:

Yan Xiao

2024年09月26日 09:35:35 -04:00

parent 9d7e7e27a5

commit dcd5a265f6

6 changed files with 1528 additions and 57 deletions

Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL

Download Patch File Download Diff File Expand all files Collapse all files

14
test/unit/common/middleware/helpers.py

View File

@@ -193,8 +193,9 @@ class FakeSwift(object):

container_existence_skip_cache = 0.0

account_existence_skip_cache = 0.0

def __init__(self, capture_unexpected_calls=True):

def __init__(self, capture_unexpected_calls=True, test_read_size=-1):

self.capture_unexpected_calls = capture_unexpected_calls

self.read_size = test_read_size

self._calls = []

self._unclosed_req_keys = defaultdict(int)

self._unread_req_paths = defaultdict(int)

@@ -325,7 +326,14 @@ class FakeSwift(object):

if (cont and not obj and method == 'UPDATE') or (

obj and method == 'PUT'):

call.body = b''.join(iter(env['wsgi.input'].read, b''))

if self.read_size < 0:

call.body = b''.join(iter(env['wsgi.input'].read, b''))

else:

call.body = b''

buf = env['wsgi.input'].read(self.read_size)

while buf:

call.body += buf

buf = env['wsgi.input'].read(self.read_size)

# simulate object PUT

if method == 'PUT' and obj:

@@ -377,7 +385,7 @@ class FakeSwift(object):

resolve_ignore_range_header(req, headers)

# range requests ought to work, hence conditional_response=True

if isinstance(body, list):

if not isinstance(body, (bytes, str)):

resp = resp_class(

req=req, headers=headers, app_iter=body,

conditional_response=req.method in ('GET', 'HEAD'),

1376
test/unit/common/middleware/test_proxy_logging.py

View File

File diff suppressed because it is too large Load Diff

12
test/unit/common/test_wsgi.py

View File

@@ -1264,6 +1264,18 @@ class TestWSGI(unittest.TestCase):

newenv = wsgi.make_env(oldenv)

self.assertIs(newenv.get('swift.infocache'), oldenv['swift.infocache'])

def test_make_env_keeps_shard_listing_history(self):

oldenv = {'swift.shard_listing_history': []}

newenv = wsgi.make_env(oldenv)

self.assertIs(newenv.get('swift.shard_listing_history'),

oldenv['swift.shard_listing_history'])

def test_make_env_keeps_base_labels(self):

oldenv = {'swift.base_labels': []}

newenv = wsgi.make_env(oldenv)

self.assertIs(newenv.get('swift.base_labels'),

oldenv['swift.base_labels'])

class CommonTestMixin(object):