Perform fewer stat calls when doing auditing of objects

Code Issues Proposed changes

Each `isdir` or `exists` call performs a stat, but that info is already
available from the exception if it doesn't exist in `listdir`.
Change-Id: I831494e3dbc8fda62ab29431471566bdb8dc6d27

This commit is contained in:

Alex Gaynor

2013年09月04日 16:00:44 -07:00

parent f1940bb05a

commit 12f95cc5a1

2 changed files with 37 additions and 9 deletions

Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL

Download Patch File Download Diff File Expand all files Collapse all files

25
swift/common/utils.py

View File

@@ -1544,25 +1544,32 @@ def audit_location_generator(devices, datadir, suffix='',

_('Skipping %s as it is not mounted'), device)

continue

datadir_path = os.path.join(devices, device, datadir)

if not os.path.exists(datadir_path):

continue

partitions = listdir(datadir_path)

for partition in partitions:

part_path = os.path.join(datadir_path, partition)

if not os.path.isdir(part_path):

try:

suffixes = listdir(part_path)

except OSError as e:

if e.errno != errno.ENOTDIR:

raise

continue

suffixes = listdir(part_path)

for asuffix in suffixes:

suff_path = os.path.join(part_path, asuffix)

if not os.path.isdir(suff_path):

try:

hashes = listdir(suff_path)

except OSError as e:

if e.errno != errno.ENOTDIR:

raise

continue

hashes = listdir(suff_path)

for hsh in hashes:

hash_path = os.path.join(suff_path, hsh)

if not os.path.isdir(hash_path):

try:

files = sorted(listdir(hash_path), reverse=True)

except OSError as e:

if e.errno != errno.ENOTDIR:

raise

continue

for fname in sorted(listdir(hash_path),

reverse=True):

for fname in files:

if suffix and not fname.endswith(suffix):

continue

path = os.path.join(hash_path, fname)

21
test/unit/common/test_utils.py

View File

@@ -2401,5 +2401,26 @@ class TestThreadpool(unittest.TestCase):

self.assertTrue(caught)

class TestAuditLocationGenerator(unittest.TestCase):

def test_non_dir_contents(self):

with temptree([]) as tmpdir:

data = os.path.join(tmpdir, "drive", "data")

os.makedirs(data)

with open(os.path.join(data, "partition1"), "w"):

pass

partition = os.path.join(data, "partition2")

os.makedirs(partition)

with open(os.path.join(partition, "suffix1"), "w"):

pass

suffix = os.path.join(partition, "suffix2")

os.makedirs(suffix)

with open(os.path.join(suffix, "hash1"), "w"):

pass

locations = utils.audit_location_generator(

tmpdir, "data", mount_check=False

)

self.assertEqual(list(locations), [])

if __name__ == '__main__':

unittest.main()