Files
a79d8508df493d5744b262e1d1830782e32dbd04
swift /bin /swift-drive-audit

215 lines
8.1 KiB
Plaintext
Raw Normal View History

2011年03月24日 15:51:44 -07:00
#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
2010年07月12日 17:03:45 -05:00
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
import glob
2010年07月12日 17:03:45 -05:00
import os
import re
import subprocess
import sys
from six.moves.configparser import ConfigParser
2010年07月12日 17:03:45 -05:00
Add new features to swift-drive-audit This patch adds two new features to swift-drive-audit. The first is an option in the drive-audit.conf file that allows the operator to prevent the drives ever being unmounted automatically, regardless of the amount of errors present. This could be of benefit in very small systems consisting of only one or two drives where the operator would like to manually unmount/fix the particular drive(s) and minimise any potential downtime. The second is another option in drive-audit.conf that allows the operator to select a recon directory. This directory will then have a drive.recon file which will keep an up-to-date record of the swift drives and any errors associated with them. An example of the output would be as follows: {"/srv/node/disk2": "0", "/srv/node/disk3": "25", "/srv/node/disk0": "0", "/srv/node/disk1": "0", "/srv/node/disk10": "0", "/srv/node/disk7": "0", "/srv/node/disk4": "137", "/srv/node/disk5": "0", "/srv/node/disk8": "0", "/srv/node/disk9": "0", "/srv/node/disk6": "0", "/srv/node/disk11": "60"} This would allow the operator to monitor the errors on the swift drives without having to spend time searching through logs. Also, if this is accepted, it should be possible to add an option to swift-recon that would keep track of this at a system level. Change-Id: Ib5dacf8622b7363e070c274c7c30c8ead448a055
2014年09月18日 17:23:54 +01:00
from swift.common.utils import backward, get_logger, dump_recon_cache, \
config_true_value
2010年07月12日 17:03:45 -05:00
2010年07月12日 17:03:45 -05:00
def get_devices(device_dir, logger):
devices = []
for line in open('/proc/mounts').readlines():
data = line.strip().split()
block_device = data[0]
mount_point = data[1]
if mount_point.startswith(device_dir):
device = {}
device['mount_point'] = mount_point
device['block_device'] = block_device
try:
device_num = os.stat(block_device).st_rdev
except OSError:
2010年07月12日 17:03:45 -05:00
# If we can't stat the device, then something weird is going on
logger.error("Error: Could not stat %s!" %
block_device)
2010年07月12日 17:03:45 -05:00
continue
device['major'] = str(os.major(device_num))
device['minor'] = str(os.minor(device_num))
devices.append(device)
for line in open('/proc/partitions').readlines()[2:]:
major, minor, blocks, kernel_device = line.strip().split()
2010年07月12日 17:03:45 -05:00
device = [d for d in devices
if d['major'] == major and d['minor'] == minor]
2010年07月12日 17:03:45 -05:00
if device:
device[0]['kernel_device'] = kernel_device
return devices
def get_errors(error_re, log_file_pattern, minutes, logger):
# Assuming log rotation is being used, we need to examine
# recently rotated files in case the rotation occurred
# just before the script is being run - the data we are
# looking for may have rotated.
#
# The globbing used before would not work with all out-of-box
# distro setup for logrotate and syslog therefore moving this
# to the config where one can set it with the desired
# globbing pattern.
log_files = [f for f in glob.glob(log_file_pattern)]
try:
log_files.sort(key=lambda f: os.stat(f).st_mtime, reverse=True)
except (IOError, OSError) as exc:
logger.error(exc)
print(exc)
sys.exit(1)
now_time = datetime.datetime.now()
end_time = now_time - datetime.timedelta(minutes=minutes)
# kern.log does not contain the year so we need to keep
# track of the year and month in case the year recently
# ticked over
year = now_time.year
prev_entry_month = now_time.month
2010年07月12日 17:03:45 -05:00
errors = {}
reached_old_logs = False
for path in log_files:
try:
f = open(path)
except IOError:
logger.error("Error: Unable to open " + path)
print("Unable to open " + path)
sys.exit(1)
for line in backward(f):
if '[ 0.000000]' in line \
or 'KERNEL supported cpus:' in line \
or 'BIOS-provided physical RAM map:' in line:
# Ignore anything before the last boot
reached_old_logs = True
break
# Solves the problem with year change - kern.log does not
# keep track of the year.
log_time_entry = line.split()[:3]
if log_time_entry[0] == 'Dec' and prev_entry_month == 'Jan':
year -= 1
prev_entry_month = log_time_entry[0]
log_time_string = '%s %s' % (year, ' '.join(log_time_entry))
try:
log_time = datetime.datetime.strptime(
log_time_string, '%Y %b %d %H:%M:%S')
except ValueError:
continue
if log_time > end_time:
for err in error_re:
for device in err.findall(line):
errors[device] = errors.get(device, 0) + 1
else:
reached_old_logs = True
break
if reached_old_logs:
break
return errors
2010年07月12日 17:03:45 -05:00
2010年07月12日 17:03:45 -05:00
def comment_fstab(mount_point):
with open('/etc/fstab', 'r') as fstab:
with open('/etc/fstab.new', 'w') as new_fstab:
for line in fstab:
parts = line.split()
if len(parts) > 2 \
and parts[1] == mount_point \
and not line.startswith('#'):
new_fstab.write('#' + line)
2010年07月12日 17:03:45 -05:00
else:
new_fstab.write(line)
os.rename('/etc/fstab.new', '/etc/fstab')
2010年07月12日 17:03:45 -05:00
if __name__ == '__main__':
c = ConfigParser()
try:
conf_path = sys.argv[1]
2011年01月26日 14:31:33 -08:00
except Exception:
print("Usage: %s CONF_FILE" % sys.argv[0].split('/')[-1])
2010年07月12日 17:03:45 -05:00
sys.exit(1)
if not c.read(conf_path):
print("Unable to read config file %s" % conf_path)
2010年07月12日 17:03:45 -05:00
sys.exit(1)
conf = dict(c.items('drive-audit'))
device_dir = conf.get('device_dir', '/srv/node')
minutes = int(conf.get('minutes', 60))
error_limit = int(conf.get('error_limit', 1))
Add new features to swift-drive-audit This patch adds two new features to swift-drive-audit. The first is an option in the drive-audit.conf file that allows the operator to prevent the drives ever being unmounted automatically, regardless of the amount of errors present. This could be of benefit in very small systems consisting of only one or two drives where the operator would like to manually unmount/fix the particular drive(s) and minimise any potential downtime. The second is another option in drive-audit.conf that allows the operator to select a recon directory. This directory will then have a drive.recon file which will keep an up-to-date record of the swift drives and any errors associated with them. An example of the output would be as follows: {"/srv/node/disk2": "0", "/srv/node/disk3": "25", "/srv/node/disk0": "0", "/srv/node/disk1": "0", "/srv/node/disk10": "0", "/srv/node/disk7": "0", "/srv/node/disk4": "137", "/srv/node/disk5": "0", "/srv/node/disk8": "0", "/srv/node/disk9": "0", "/srv/node/disk6": "0", "/srv/node/disk11": "60"} This would allow the operator to monitor the errors on the swift drives without having to spend time searching through logs. Also, if this is accepted, it should be possible to add an option to swift-recon that would keep track of this at a system level. Change-Id: Ib5dacf8622b7363e070c274c7c30c8ead448a055
2014年09月18日 17:23:54 +01:00
recon_cache_path = conf.get('recon_cache_path', "/var/cache/swift")
log_file_pattern = conf.get('log_file_pattern',
'/var/log/kern.*[!.][!g][!z]')
log_to_console = config_true_value(conf.get('log_to_console', False))
error_re = []
for conf_key in conf:
if conf_key.startswith('regex_pattern_'):
error_pattern = conf[conf_key]
try:
r = re.compile(error_pattern)
except re.error:
sys.exit('Error: unable to compile regex pattern "%s"' %
error_pattern)
error_re.append(r)
if not error_re:
error_re = [
re.compile(r'\berror\b.*\b(sd[a-z]{1,2}\d?)\b'),
re.compile(r'\b(sd[a-z]{1,2}\d?)\b.*\berror\b'),
]
2011年02月10日 00:01:07 -08:00
conf['log_name'] = conf.get('log_name', 'drive-audit')
logger = get_logger(conf, log_to_console=log_to_console,
log_route='drive-audit')
2010年07月12日 17:03:45 -05:00
devices = get_devices(device_dir, logger)
logger.debug("Devices found: %s" % str(devices))
if not devices:
logger.error("Error: No devices found!")
Add new features to swift-drive-audit This patch adds two new features to swift-drive-audit. The first is an option in the drive-audit.conf file that allows the operator to prevent the drives ever being unmounted automatically, regardless of the amount of errors present. This could be of benefit in very small systems consisting of only one or two drives where the operator would like to manually unmount/fix the particular drive(s) and minimise any potential downtime. The second is another option in drive-audit.conf that allows the operator to select a recon directory. This directory will then have a drive.recon file which will keep an up-to-date record of the swift drives and any errors associated with them. An example of the output would be as follows: {"/srv/node/disk2": "0", "/srv/node/disk3": "25", "/srv/node/disk0": "0", "/srv/node/disk1": "0", "/srv/node/disk10": "0", "/srv/node/disk7": "0", "/srv/node/disk4": "137", "/srv/node/disk5": "0", "/srv/node/disk8": "0", "/srv/node/disk9": "0", "/srv/node/disk6": "0", "/srv/node/disk11": "60"} This would allow the operator to monitor the errors on the swift drives without having to spend time searching through logs. Also, if this is accepted, it should be possible to add an option to swift-recon that would keep track of this at a system level. Change-Id: Ib5dacf8622b7363e070c274c7c30c8ead448a055
2014年09月18日 17:23:54 +01:00
recon_errors = {}
Add swift-recon feature to track swift-drive-audit error count This is a follow-on from a previous commit which added recon info for swift-drive-audit (https://review.openstack.org/#/c/122468/). Here, the "--drievaudit" option is added to swift-recon tool. This feature gives the statistics for the system-wide drive errors flagged by swift-drive-audit. An example of the output is as follows: (verbose mode) swift-recon --driveaudit -v =============================================================================== --> Starting reconnaissance on 5 hosts =============================================================================== [2015年03月11日 17:13:39] Checking drive-audit errors -> http://1.2.3.4:6000/recon/driveaudit: {'drive_audit_errors': 14} -> http://1.2.3.5:6000/recon/driveaudit: {'drive_audit_errors': 0} -> http://1.2.3.6:6000/recon/driveaudit: {'drive_audit_errors': 37} -> http://1.2.3.7:6000/recon/driveaudit: {'drive_audit_errors': 101} -> http://1.2.3.8:6000/recon/driveaudit: {'drive_audit_errors': 0} [drive_audit_errors] low: 0, high: 101, avg: 30.4, total: 152, Failed: 0.0%, no_result: 0, reported: 5 =============================================================================== Change-Id: Ia16c52a9d613eeb3de1a5a428d88dd1233631912
2015年03月12日 15:40:39 +00:00
total_errors = 0
Add new features to swift-drive-audit This patch adds two new features to swift-drive-audit. The first is an option in the drive-audit.conf file that allows the operator to prevent the drives ever being unmounted automatically, regardless of the amount of errors present. This could be of benefit in very small systems consisting of only one or two drives where the operator would like to manually unmount/fix the particular drive(s) and minimise any potential downtime. The second is another option in drive-audit.conf that allows the operator to select a recon directory. This directory will then have a drive.recon file which will keep an up-to-date record of the swift drives and any errors associated with them. An example of the output would be as follows: {"/srv/node/disk2": "0", "/srv/node/disk3": "25", "/srv/node/disk0": "0", "/srv/node/disk1": "0", "/srv/node/disk10": "0", "/srv/node/disk7": "0", "/srv/node/disk4": "137", "/srv/node/disk5": "0", "/srv/node/disk8": "0", "/srv/node/disk9": "0", "/srv/node/disk6": "0", "/srv/node/disk11": "60"} This would allow the operator to monitor the errors on the swift drives without having to spend time searching through logs. Also, if this is accepted, it should be possible to add an option to swift-recon that would keep track of this at a system level. Change-Id: Ib5dacf8622b7363e070c274c7c30c8ead448a055
2014年09月18日 17:23:54 +01:00
for device in devices:
recon_errors[device['mount_point']] = 0
errors = get_errors(error_re, log_file_pattern, minutes, logger)
2010年07月12日 17:03:45 -05:00
logger.debug("Errors found: %s" % str(errors))
unmounts = 0
for kernel_device, count in errors.items():
2010年07月12日 17:03:45 -05:00
if count >= error_limit:
device = \
[d for d in devices if d['kernel_device'] == kernel_device]
2010年07月12日 17:03:45 -05:00
if device:
mount_point = device[0]['mount_point']
if mount_point.startswith(device_dir):
Add new features to swift-drive-audit This patch adds two new features to swift-drive-audit. The first is an option in the drive-audit.conf file that allows the operator to prevent the drives ever being unmounted automatically, regardless of the amount of errors present. This could be of benefit in very small systems consisting of only one or two drives where the operator would like to manually unmount/fix the particular drive(s) and minimise any potential downtime. The second is another option in drive-audit.conf that allows the operator to select a recon directory. This directory will then have a drive.recon file which will keep an up-to-date record of the swift drives and any errors associated with them. An example of the output would be as follows: {"/srv/node/disk2": "0", "/srv/node/disk3": "25", "/srv/node/disk0": "0", "/srv/node/disk1": "0", "/srv/node/disk10": "0", "/srv/node/disk7": "0", "/srv/node/disk4": "137", "/srv/node/disk5": "0", "/srv/node/disk8": "0", "/srv/node/disk9": "0", "/srv/node/disk6": "0", "/srv/node/disk11": "60"} This would allow the operator to monitor the errors on the swift drives without having to spend time searching through logs. Also, if this is accepted, it should be possible to add an option to swift-recon that would keep track of this at a system level. Change-Id: Ib5dacf8622b7363e070c274c7c30c8ead448a055
2014年09月18日 17:23:54 +01:00
if config_true_value(conf.get('unmount_failed_device',
True)):
logger.info("Unmounting %s with %d errors" %
(mount_point, count))
subprocess.call(['umount', '-fl', mount_point])
logger.info("Commenting out %s from /etc/fstab" %
(mount_point))
comment_fstab(mount_point)
unmounts += 1
else:
logger.info("Detected %s with %d errors "
"(Device not unmounted)" %
(mount_point, count))
Add new features to swift-drive-audit This patch adds two new features to swift-drive-audit. The first is an option in the drive-audit.conf file that allows the operator to prevent the drives ever being unmounted automatically, regardless of the amount of errors present. This could be of benefit in very small systems consisting of only one or two drives where the operator would like to manually unmount/fix the particular drive(s) and minimise any potential downtime. The second is another option in drive-audit.conf that allows the operator to select a recon directory. This directory will then have a drive.recon file which will keep an up-to-date record of the swift drives and any errors associated with them. An example of the output would be as follows: {"/srv/node/disk2": "0", "/srv/node/disk3": "25", "/srv/node/disk0": "0", "/srv/node/disk1": "0", "/srv/node/disk10": "0", "/srv/node/disk7": "0", "/srv/node/disk4": "137", "/srv/node/disk5": "0", "/srv/node/disk8": "0", "/srv/node/disk9": "0", "/srv/node/disk6": "0", "/srv/node/disk11": "60"} This would allow the operator to monitor the errors on the swift drives without having to spend time searching through logs. Also, if this is accepted, it should be possible to add an option to swift-recon that would keep track of this at a system level. Change-Id: Ib5dacf8622b7363e070c274c7c30c8ead448a055
2014年09月18日 17:23:54 +01:00
recon_errors[mount_point] = count
Add swift-recon feature to track swift-drive-audit error count This is a follow-on from a previous commit which added recon info for swift-drive-audit (https://review.openstack.org/#/c/122468/). Here, the "--drievaudit" option is added to swift-recon tool. This feature gives the statistics for the system-wide drive errors flagged by swift-drive-audit. An example of the output is as follows: (verbose mode) swift-recon --driveaudit -v =============================================================================== --> Starting reconnaissance on 5 hosts =============================================================================== [2015年03月11日 17:13:39] Checking drive-audit errors -> http://1.2.3.4:6000/recon/driveaudit: {'drive_audit_errors': 14} -> http://1.2.3.5:6000/recon/driveaudit: {'drive_audit_errors': 0} -> http://1.2.3.6:6000/recon/driveaudit: {'drive_audit_errors': 37} -> http://1.2.3.7:6000/recon/driveaudit: {'drive_audit_errors': 101} -> http://1.2.3.8:6000/recon/driveaudit: {'drive_audit_errors': 0} [drive_audit_errors] low: 0, high: 101, avg: 30.4, total: 152, Failed: 0.0%, no_result: 0, reported: 5 =============================================================================== Change-Id: Ia16c52a9d613eeb3de1a5a428d88dd1233631912
2015年03月12日 15:40:39 +00:00
total_errors += count
Add new features to swift-drive-audit This patch adds two new features to swift-drive-audit. The first is an option in the drive-audit.conf file that allows the operator to prevent the drives ever being unmounted automatically, regardless of the amount of errors present. This could be of benefit in very small systems consisting of only one or two drives where the operator would like to manually unmount/fix the particular drive(s) and minimise any potential downtime. The second is another option in drive-audit.conf that allows the operator to select a recon directory. This directory will then have a drive.recon file which will keep an up-to-date record of the swift drives and any errors associated with them. An example of the output would be as follows: {"/srv/node/disk2": "0", "/srv/node/disk3": "25", "/srv/node/disk0": "0", "/srv/node/disk1": "0", "/srv/node/disk10": "0", "/srv/node/disk7": "0", "/srv/node/disk4": "137", "/srv/node/disk5": "0", "/srv/node/disk8": "0", "/srv/node/disk9": "0", "/srv/node/disk6": "0", "/srv/node/disk11": "60"} This would allow the operator to monitor the errors on the swift drives without having to spend time searching through logs. Also, if this is accepted, it should be possible to add an option to swift-recon that would keep track of this at a system level. Change-Id: Ib5dacf8622b7363e070c274c7c30c8ead448a055
2014年09月18日 17:23:54 +01:00
recon_file = recon_cache_path + "/drive.recon"
dump_recon_cache(recon_errors, recon_file, logger)
Add swift-recon feature to track swift-drive-audit error count This is a follow-on from a previous commit which added recon info for swift-drive-audit (https://review.openstack.org/#/c/122468/). Here, the "--drievaudit" option is added to swift-recon tool. This feature gives the statistics for the system-wide drive errors flagged by swift-drive-audit. An example of the output is as follows: (verbose mode) swift-recon --driveaudit -v =============================================================================== --> Starting reconnaissance on 5 hosts =============================================================================== [2015年03月11日 17:13:39] Checking drive-audit errors -> http://1.2.3.4:6000/recon/driveaudit: {'drive_audit_errors': 14} -> http://1.2.3.5:6000/recon/driveaudit: {'drive_audit_errors': 0} -> http://1.2.3.6:6000/recon/driveaudit: {'drive_audit_errors': 37} -> http://1.2.3.7:6000/recon/driveaudit: {'drive_audit_errors': 101} -> http://1.2.3.8:6000/recon/driveaudit: {'drive_audit_errors': 0} [drive_audit_errors] low: 0, high: 101, avg: 30.4, total: 152, Failed: 0.0%, no_result: 0, reported: 5 =============================================================================== Change-Id: Ia16c52a9d613eeb3de1a5a428d88dd1233631912
2015年03月12日 15:40:39 +00:00
dump_recon_cache({'drive_audit_errors': total_errors}, recon_file, logger)
Add new features to swift-drive-audit This patch adds two new features to swift-drive-audit. The first is an option in the drive-audit.conf file that allows the operator to prevent the drives ever being unmounted automatically, regardless of the amount of errors present. This could be of benefit in very small systems consisting of only one or two drives where the operator would like to manually unmount/fix the particular drive(s) and minimise any potential downtime. The second is another option in drive-audit.conf that allows the operator to select a recon directory. This directory will then have a drive.recon file which will keep an up-to-date record of the swift drives and any errors associated with them. An example of the output would be as follows: {"/srv/node/disk2": "0", "/srv/node/disk3": "25", "/srv/node/disk0": "0", "/srv/node/disk1": "0", "/srv/node/disk10": "0", "/srv/node/disk7": "0", "/srv/node/disk4": "137", "/srv/node/disk5": "0", "/srv/node/disk8": "0", "/srv/node/disk9": "0", "/srv/node/disk6": "0", "/srv/node/disk11": "60"} This would allow the operator to monitor the errors on the swift drives without having to spend time searching through logs. Also, if this is accepted, it should be possible to add an option to swift-recon that would keep track of this at a system level. Change-Id: Ib5dacf8622b7363e070c274c7c30c8ead448a055
2014年09月18日 17:23:54 +01:00
2010年07月12日 17:03:45 -05:00
if unmounts == 0:
logger.info("No drives were unmounted")