Differentiate unlinks and outdated unlinks

Code Issues Proposed changes

This commit differentiates async pendings that are unlinked due to a successful object update (unlinks)from those async pendings that are unlinked due to a newer async_pending existing for the same object (outdated_unlinks).
Change-Id: I66e16207f3e368248617fc7ed3c6b5c80c54b1b5

This commit is contained in:

Chinemerem

2024年11月01日 08:08:01 -07:00

committed by Chinemerem Chigbo

parent a8ad97f838

commit 39e4ae3076

2 changed files with 39 additions and 23 deletions

Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL

Download Patch File Download Diff File Expand all files Collapse all files

21
swift/obj/updater.py

View File

@@ -229,12 +229,14 @@ class SweepStats(object):

skips / (skips + successes + failures)

"""

def __init__(self, errors=0, failures=0, quarantines=0, successes=0,

unlinks=0, redirects=0, skips=0, deferrals=0, drains=0):

unlinks=0, outdated_unlinks=0, redirects=0, skips=0,

deferrals=0, drains=0):

self.errors = errors

self.failures = failures

self.quarantines = quarantines

self.successes = successes

self.unlinks = unlinks

self.outdated_unlinks = outdated_unlinks

self.redirects = redirects

self.skips = skips

self.deferrals = deferrals

@@ -242,8 +244,9 @@ class SweepStats(object):

def copy(self):

return type(self)(self.errors, self.failures, self.quarantines,

self.successes, self.unlinks, self.redirects,

self.skips, self.deferrals, self.drains)

self.successes, self.unlinks, self.outdated_unlinks,

self.redirects, self.skips, self.deferrals,

self.drains)

def since(self, other):

return type(self)(self.errors - other.errors,

@@ -251,6 +254,7 @@ class SweepStats(object):

self.quarantines - other.quarantines,

self.successes - other.successes,

self.unlinks - other.unlinks,

self.outdated_unlinks - other.outdated_unlinks,

self.redirects - other.redirects,

self.skips - other.skips,

self.deferrals - other.deferrals,

@@ -262,6 +266,7 @@ class SweepStats(object):

self.quarantines = 0

self.successes = 0

self.unlinks = 0

self.outdated_unlinks = 0

self.redirects = 0

self.skips = 0

self.deferrals = 0

@@ -273,6 +278,7 @@ class SweepStats(object):

(self.failures, 'failures'),

(self.quarantines, 'quarantines'),

(self.unlinks, 'unlinks'),

(self.outdated_unlinks, 'outdated_unlinks'),

(self.errors, 'errors'),

(self.redirects, 'redirects'),

(self.skips, 'skips'),

@@ -520,8 +526,8 @@ class ObjectUpdater(Daemon):

#

# This way, our caller only gets useful async_pendings.

if obj_hash == last_obj_hash:

self.stats.unlinks += 1

self.logger.increment('unlinks')

self.stats.outdated_unlinks += 1

self.logger.increment('outdated_unlinks')

try:

os.unlink(update_path)

except OSError as e:

@@ -583,7 +589,9 @@ class ObjectUpdater(Daemon):

'in %(elapsed).02fs seconds:, '

'%(successes)d successes, %(failures)d failures, '

'%(quarantines)d quarantines, '

'%(unlinks)d unlinks, %(errors)d errors, '

'%(unlinks)d unlinks, '

'%(outdated_unlinks)d outdated_unlinks, '

'%(errors)d errors, '

'%(redirects)d redirects, '

'%(skips)d skips, '

'%(deferrals)d deferrals, '

@@ -596,6 +604,7 @@ class ObjectUpdater(Daemon):

'failures': sweep_totals.failures,

'quarantines': sweep_totals.quarantines,

'unlinks': sweep_totals.unlinks,

'outdated_unlinks': sweep_totals.outdated_unlinks,

'errors': sweep_totals.errors,

'redirects': sweep_totals.redirects,

'skips': sweep_totals.skips,

41
test/unit/obj/test_updater.py

View File

@@ -354,22 +354,22 @@ class TestObjectUpdater(unittest.TestCase):

self.assertIn("sweep progress", info_lines[1])

# the space ensures it's a positive number

self.assertIn(

"2 successes, 0 failures, 0 quarantines, 2 unlinks, 0 errors, "

"0 redirects",

"2 successes, 0 failures, 0 quarantines, 2 unlinks, "

"0 outdated_unlinks, 0 errors, 0 redirects",

info_lines[1])

self.assertIn(self.sda1, info_lines[1])

self.assertIn("sweep progress", info_lines[2])

self.assertIn(

"4 successes, 0 failures, 0 quarantines, 4 unlinks, 0 errors, "

"0 redirects",

"4 successes, 0 failures, 0 quarantines, 4 unlinks, "

"0 outdated_unlinks, 0 errors, 0 redirects",

info_lines[2])

self.assertIn(self.sda1, info_lines[2])

self.assertIn("sweep complete", info_lines[3])

self.assertIn(

"5 successes, 0 failures, 0 quarantines, 5 unlinks, 0 errors, "

"0 redirects",

"5 successes, 0 failures, 0 quarantines, 5 unlinks, "

"0 outdated_unlinks, 0 errors, 0 redirects",

info_lines[3])

self.assertIn(self.sda1, info_lines[3])

@@ -418,8 +418,8 @@ class TestObjectUpdater(unittest.TestCase):

self.assertEqual(len(completion_lines), 1)

self.assertIn("sweep complete", completion_lines[0])

self.assertIn(

"6 successes, 0 failures, 0 quarantines, 6 unlinks, 0 errors, "

"0 redirects",

"6 successes, 0 failures, 0 quarantines, 6 unlinks, "

"0 outdated_unlinks, 0 errors, 0 redirects",

completion_lines[0])

@mock.patch.object(object_updater, 'check_drive')

@@ -525,7 +525,7 @@ class TestObjectUpdater(unittest.TestCase):

self.assertTrue(not os.path.exists(older_op_path))

self.assertTrue(os.path.exists(op_path))

self.assertEqual(ou.logger.statsd_client.get_increment_counts(),

{'failures': 1, 'unlinks': 1})

{'failures': 1, 'outdated_unlinks': 1})

self.assertIsNone(pickle.load(open(op_path, 'rb')).get('successes'))

self.assertEqual(

['ERROR with remote server 127.0.0.1:67890/sda1: '

@@ -1400,7 +1400,8 @@ class TestObjectUpdater(unittest.TestCase):

info_lines = self.logger.get_lines_for_level('info')

self.assertTrue(info_lines)

self.assertIn('2 successes, 0 failures, 0 quarantines, 2 unlinks, '

'0 errors, 0 redirects, 9 skips, 9 deferrals, 0 drains',

'0 outdated_unlinks, 0 errors, 0 redirects, 9 skips, '

'9 deferrals, 0 drains',

info_lines[-1])

self.assertEqual({'skips': 9, 'successes': 2, 'unlinks': 2,

'deferrals': 9},

@@ -1439,7 +1440,8 @@ class TestObjectUpdater(unittest.TestCase):

info_lines = self.logger.get_lines_for_level('info')

self.assertTrue(info_lines)

self.assertIn('11 successes, 0 failures, 0 quarantines, 11 unlinks, '

'0 errors, 0 redirects, 0 skips, 0 deferrals, 0 drains',

'0 outdated_unlinks, 0 errors, 0 redirects, 0 skips, '

'0 deferrals, 0 drains',

info_lines[-1])

self.assertEqual({'successes': 11, 'unlinks': 11},

self.logger.statsd_client.get_increment_counts())

@@ -1507,7 +1509,8 @@ class TestObjectUpdater(unittest.TestCase):

info_lines = self.logger.get_lines_for_level('info')

self.assertTrue(info_lines)

self.assertIn('2 successes, 0 failures, 0 quarantines, 2 unlinks, '

'0 errors, 0 redirects, 2 skips, 2 deferrals, 0 drains',

'0 outdated_unlinks, 0 errors, 0 redirects, 2 skips, '

'2 deferrals, 0 drains',

info_lines[-1])

self.assertEqual({'skips': 2, 'successes': 2, 'unlinks': 2,

'deferrals': 2},

@@ -1625,7 +1628,8 @@ class TestObjectUpdater(unittest.TestCase):

info_lines = self.logger.get_lines_for_level('info')

self.assertTrue(info_lines)

self.assertIn('3 successes, 0 failures, 0 quarantines, 3 unlinks, '

'0 errors, 0 redirects, 1 skips, 2 deferrals, 1 drains',

'0 outdated_unlinks, 0 errors, 0 redirects, 1 skips, '

'2 deferrals, 1 drains',

info_lines[-1])

self.assertEqual(

{'skips': 1, 'successes': 3, 'unlinks': 3, 'deferrals': 2,

@@ -1746,7 +1750,8 @@ class TestObjectUpdater(unittest.TestCase):

info_lines = self.logger.get_lines_for_level('info')

self.assertTrue(info_lines)

self.assertIn('4 successes, 0 failures, 0 quarantines, 4 unlinks, '

'0 errors, 0 redirects, 1 skips, 3 deferrals, 2 drains',

'0 outdated_unlinks, 0 errors, 0 redirects, 1 skips, '

'3 deferrals, 2 drains',

info_lines[-1])

self.assertEqual(

{'skips': 1, 'successes': 4, 'unlinks': 4, 'deferrals': 3,

@@ -1874,7 +1879,8 @@ class TestObjectUpdater(unittest.TestCase):

info_lines = self.logger.get_lines_for_level('info')

self.assertTrue(info_lines)

self.assertIn('5 successes, 0 failures, 0 quarantines, 5 unlinks, '

'0 errors, 0 redirects, 2 skips, 4 deferrals, 2 drains',

'0 outdated_unlinks, 0 errors, 0 redirects, 2 skips, '

'4 deferrals, 2 drains',

info_lines[-1])

self.assertEqual(

{'successes': 5, 'unlinks': 5, 'deferrals': 4, 'drains': 2},

@@ -2181,8 +2187,9 @@ class TestSweepStats(unittest.TestCase):

num_props = len(vars(object_updater.SweepStats()))

stats = object_updater.SweepStats(*range(1, num_props + 1))

self.assertEqual(

'4 successes, 2 failures, 3 quarantines, 5 unlinks, 1 errors, '

'6 redirects, 7 skips, 8 deferrals, 9 drains', str(stats))

'4 successes, 2 failures, 3 quarantines, 5 unlinks, '

'6 outdated_unlinks, 1 errors, 7 redirects, 8 skips, 9 deferrals, '

'10 drains', str(stats))

if __name__ == '__main__':