sharder: reduce rate of 'Reclaimable db...' warnings

Code Issues Proposed changes

Sharded DBs that have been completely emptied of objects will not be
removed if auto-sharding is disabled. The shards remain in place and
the sharder will repeatedly warn of a 'Reclaimable db stuck waiting
for shrinking'. This can occur often if expiring objects containers
become sharded, since these containers will always, by design, be
emptied. Each time this happens, another warning log starts to be
repeatedly emitted.
This patch reduces the rate of this specific warning to once per 24
hours.
Change-Id: I493cd661fe98817ca5fd2daf2a681fdf69e28923

This commit is contained in:

Alistair Coles

2023年10月18日 15:30:51 +01:00

parent 195f54ecb5

commit f14b1de4d5

2 changed files with 69 additions and 4 deletions

Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL

Download Patch File Download Diff File Expand all files Collapse all files

31
swift/container/sharder.py

View File

@@ -50,6 +50,8 @@ CLEAVE_SUCCESS = 0

CLEAVE_FAILED = 1

CLEAVE_EMPTY = 2

DEFAULT_PERIODIC_WARNINGS_INTERVAL = 24 * 3600

def sharding_enabled(broker):

# NB all shards will by default have been created with

@@ -908,9 +910,13 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):

(internal_client_conf_path, err))

self.stats_interval = float(conf.get('stats_interval', '3600'))

self.reported = 0

self.periodic_warnings_interval = float(

conf.get('periodic_warnings_interval',

DEFAULT_PERIODIC_WARNINGS_INTERVAL))

self.periodic_warnings_start = time.time()

self.periodic_warnings = set()

def _format_log_msg(self, broker, msg, *args):

# make best effort to include broker properties...

def _get_broker_details(self, broker):

try:

db_file = broker.db_file

except Exception: # noqa

@@ -919,7 +925,11 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):

path = broker.path

except Exception: # noqa

path = ''

return db_file, path

def _format_log_msg(self, broker, msg, *args):

# make best effort to include broker properties...

db_file, path = self._get_broker_details(broker)

if args:

msg = msg % args

return '%s, path: %s, db: %s' % (msg, quote(path), db_file)

@@ -939,6 +949,19 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):

def warning(self, broker, msg, *args, **kwargs):

self._log(logging.WARNING, broker, msg, *args, **kwargs)

def periodic_warning(self, broker, msg, *args, **kwargs):

now = time.time()

if now - self.periodic_warnings_start >= \

self.periodic_warnings_interval:

self.periodic_warnings.clear()

self.periodic_warnings_start = now

db_file, path = self._get_broker_details(broker)

key = (db_file, msg)

if key not in self.periodic_warnings:

self.periodic_warnings.add(key)

self._log(logging.WARNING, broker, msg, *args, **kwargs)

def error(self, broker, msg, *args, **kwargs):

self._log(logging.ERROR, broker, msg, *args, **kwargs)

@@ -1549,8 +1572,8 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):

if broker.is_deleted():

if broker.is_old_enough_to_reclaim(time.time(), self.reclaim_age) \

and not broker.is_empty_enough_to_reclaim():

self.warning(broker,

'Reclaimable db stuck waiting for shrinking')

self.periodic_warning(

broker, 'Reclaimable db stuck waiting for shrinking')

# if the container has been marked as deleted, all metadata will

# have been erased so no point auditing. But we want it to pass, in

# case any objects exist inside it.

42
test/unit/container/test_sharder.py

View File

@@ -517,6 +517,48 @@ class TestSharder(BaseTestSharder):

do_test('warning')

do_test('error')

def test_periodic_warning(self):

now = [time.time()]

def mock_time():

return now[0]

with mock.patch('swift.container.sharder.time.time', mock_time):

with self._mock_sharder() as sharder:

sharder.periodic_warnings_interval = 5

broker1 = self._make_broker(container='c1')

broker2 = self._make_broker(container='c2')

for i in range(5):

sharder.periodic_warning(broker1, 'periodic warning 1')

sharder.periodic_warning(broker1, 'periodic warning 1a')

sharder.periodic_warning(broker2, 'periodic warning 2')

now[0] += 1

sharder.warning(broker1, 'normal warning')

sharder.periodic_warning(broker1, 'periodic warning 1')

sharder.periodic_warning(broker1, 'periodic warning 1a')

sharder.periodic_warning(broker2, 'periodic warning 2')

sharder.warning(broker1, 'normal warning')

for i in range(10):

sharder.periodic_warning(broker1, 'periodic warning 1')

sharder.periodic_warning(broker1, 'periodic warning 1a')

sharder.periodic_warning(broker2, 'periodic warning 2')

now[0] += 1

lines = self.logger.get_lines_for_level('warning')

self.assertEqual(11, len(lines))

self.assertEqual(

['periodic warning 1, path: a/c1, db: %s' % broker1.db_file,

'periodic warning 1a, path: a/c1, db: %s' % broker1.db_file,

'periodic warning 2, path: a/c2, db: %s' % broker2.db_file,

'normal warning, path: a/c1, db: %s' % broker1.db_file,

'periodic warning 1, path: a/c1, db: %s' % broker1.db_file,

'periodic warning 1a, path: a/c1, db: %s' % broker1.db_file,

'periodic warning 2, path: a/c2, db: %s' % broker2.db_file,

'normal warning, path: a/c1, db: %s' % broker1.db_file,

'periodic warning 1, path: a/c1, db: %s' % broker1.db_file,

'periodic warning 1a, path: a/c1, db: %s' % broker1.db_file,

'periodic warning 2, path: a/c2, db: %s' % broker2.db_file],

lines)

def _assert_stats(self, expected, sharder, category):

# assertEqual doesn't work with a stats defaultdict so copy to a dict

# before comparing