Files
8ab8f1ca455df315677cff66c3de6c2279ce90ee
devstack /tools /worlddump.py

202 lines
5.3 KiB
Python
Raw Normal View History

#!/usr/bin/env python
#
# Copyright 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Dump the state of the world for post mortem."""
import argparse
import datetime
from distutils import spawn
import fnmatch
import os
import os.path
import subprocess
import sys
def get_options():
parser = argparse.ArgumentParser(
description='Dump world state for debugging')
parser.add_argument('-d', '--dir',
default='.',
help='Output directory for worlddump')
parser.add_argument('-n', '--name',
default='',
help='Additional name to tag into file')
return parser.parse_args()
def filename(dirname, name=""):
now = datetime.datetime.utcnow()
fmt = "worlddump-%Y-%m-%d-%H%M%S"
if name:
fmt += "-" + name
fmt += ".txt"
return os.path.join(dirname, now.strftime(fmt))
def warn(msg):
print "WARN: %s" % msg
def _dump_cmd(cmd):
print cmd
print "-" * len(cmd)
print
try:
subprocess.check_call(cmd, shell=True)
print
except subprocess.CalledProcessError as e:
print "*** Failed to run '%(cmd)s': %(err)s" % {'cmd': cmd, 'err': e}
def _find_cmd(cmd):
if not spawn.find_executable(cmd):
print "*** %s not found: skipping" % cmd
return False
return True
def _header(name):
print
print name
print "=" * len(name)
print
def disk_space():
# the df output
_header("File System Summary")
dfraw = os.popen("df -Ph").read()
df = [s.split() for s in dfraw.splitlines()]
for fs in df:
try:
if int(fs[4][:-1]) > 95:
warn("Device %s (%s) is %s full, might be an issue" % (
fs[0], fs[5], fs[4]))
except ValueError:
# if it doesn't look like an int, that's fine
pass
print dfraw
def ebtables_dump():
tables = ['filter', 'nat', 'broute']
_header("EB Tables Dump")
if not _find_cmd('ebtables'):
return
for table in tables:
_dump_cmd("sudo ebtables -t %s -L" % table)
def iptables_dump():
tables = ['filter', 'nat', 'mangle']
_header("IP Tables Dump")
for table in tables:
_dump_cmd("sudo iptables --line-numbers -L -nv -t %s" % table)
def _netns_list():
process = subprocess.Popen(['ip', 'netns'], stdout=subprocess.PIPE)
stdout, _ = process.communicate()
return stdout.split()
def network_dump():
_header("Network Dump")
_dump_cmd("brctl show")
_dump_cmd("arp -n")
ip_cmds = ["addr", "link", "route"]
for cmd in ip_cmds + ['netns']:
_dump_cmd("ip %s" % cmd)
for netns_ in _netns_list():
for cmd in ip_cmds:
args = {'netns': netns_, 'cmd': cmd}
_dump_cmd('sudo ip netns exec %(netns)s ip %(cmd)s' % args)
def ovs_dump():
_header("Open vSwitch Dump")
# NOTE(cdent): If we're not using neutron + ovs these commands
# will not be present so
if not _find_cmd('ovs-vsctl'):
return
# NOTE(ihrachys): worlddump is used outside of devstack context (f.e. in
# grenade), so there is no single place to determine the bridge names from.
# Hardcode for now.
bridges = ('br-int', 'br-tun', 'br-ex')
_dump_cmd("sudo ovs-vsctl show")
for bridge in bridges:
_dump_cmd("sudo ovs-ofctl show %s" % bridge)
for bridge in bridges:
_dump_cmd("sudo ovs-ofctl dump-flows %s" % bridge)
def process_list():
_header("Process Listing")
_dump_cmd("ps axo "
"user,ppid,pid,pcpu,pmem,vsz,rss,tty,stat,start,time,args")
def compute_consoles():
_header("Compute consoles")
for root, dirnames, filenames in os.walk('/opt/stack'):
for filename in fnmatch.filter(filenames, 'console.log'):
fullpath = os.path.join(root, filename)
_dump_cmd("sudo cat %s" % fullpath)
def guru_meditation_report():
_header("nova-compute Guru Meditation Report")
try:
subprocess.check_call(["pgrep","nova-compute"])
except subprocess.CalledProcessError:
print "Skipping as nova-compute does not appear to be running"
return
worlddump: Use SIGUSR2 instead of SIGUSR1 The function guru_meditation_report() currently uses the User-defined signal SIGUSR1 to kill a Nova Compute process so that a Guru Meditation Report is generated. Testing locally, in a DevStack instance, manually attempting to kill a Nova compute process [kill -s USR1 `pgrep nova-compute`] does not result in process being terminated, and no error report generated. It turns out[1] that SIGUSR1 is used by Apache 'mod_wsgi'. Using the signal SIGUSR2 resolves this issue (i.e. 'nova-compute' process is terminated, and the Guru Meditation Report is generated). So, use USR2, instead of USR1. Corresponding oslo.reports related commit[2]. [1] https://code.google.com/p/modwsgi/wiki/ConfigurationDirectives#WSGIRestrictSignal [2] 45b1c02d113051d147e54ef921ce8e94135542d8 -- guru_meditation_report: Use SIGUSR2 instead of SIGUSR1 [3] Original DevStack commit that brought in this change -- 2ebe993b25462919e8aeeb896c9f91b6be7aa573 Change-Id: I8a7eaf71b83edca3c80074d6bf2d471e3db6142b
2015年09月14日 13:17:56 +02:00
_dump_cmd("kill -s USR2 `pgrep nova-compute`")
print "guru meditation report in nova-compute log"
def main():
opts = get_options()
fname = filename(opts.dir, opts.name)
print "World dumping... see %s for details" % fname
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
with open(fname, 'w') as f:
os.dup2(f.fileno(), sys.stdout.fileno())
disk_space()
process_list()
network_dump()
ovs_dump()
iptables_dump()
ebtables_dump()
compute_consoles()
guru_meditation_report()
if __name__ == '__main__':
try:
sys.exit(main())
except KeyboardInterrupt:
sys.exit(1)