Grab additional data about IO to profile service timeouts

Code Issues Proposed changes

We are hitting timeouts in IO bound processes as swift but we
don't have much information about what processes are causing
this IO peaks.
This patch adds iotop and iostat information to the logs in
addition to dstat output.
Change-Id: I4c50d4405a923eb2652fc39f6cea9889597ae44e

This commit is contained in:

Alfredo Moralejo

2017年03月07日 14:49:01 +01:00

parent 9373311dfe

commit 423c3d1058

2 changed files with 22 additions and 2 deletions

Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL

Download Patch File Download Diff File Expand all files Collapse all files

10
copy_logs.sh

View File

@@ -114,6 +114,16 @@ if [ -f /var/log/dstat.log ] ; then

sudo cp /var/log/dstat.log $LOG_DIR/

fi

# iostat logs

if [ -f /var/log/iostat.log ] ; then

sudo cp /var/log/iostat.log $LOG_DIR/

fi

# iotop logs

if [ -f /var/log/iotop.log ] ; then

sudo cp /var/log/iotop.log $LOG_DIR/

fi

# libvirt

if [ -d /var/log/libvirt ] ; then

sudo cp -r /var/log/libvirt $LOG_DIR/

14
run_tests.sh

View File

@@ -140,9 +140,9 @@ if [ "${MANAGE_HIERA}" = true ]; then

fi

if uses_debs; then

$SUDO apt-get install -y dstat ebtables

$SUDO apt-get install -y dstat ebtables iotop sysstat

elif is_fedora; then

$SUDO yum install -y dstat setools setroubleshoot audit

$SUDO yum install -y dstat setools setroubleshoot audit iotop sysstat

$SUDO service auditd start

# SElinux in permissive mode so later we can catch alerts

$SUDO selinuxenabled && $SUDO setenforce 0

@@ -154,6 +154,16 @@ if type "dstat" 2>/dev/null; then

$SUDO dstat -tcmndrylpg --top-cpu-adv --top-io-adv --nocolor | $SUDO tee --append /var/log/dstat.log > /dev/null &

fi

if type "iostat" 2>/dev/null; then

print_header 'Start iostat'

$SUDO iostat -x -k -d -t 4 | $SUDO tee --append /var/log/iostat.log > /dev/null &

fi

if [ -f "/usr/sbin/iotop" ]; then

print_header 'Start iotop'

 $SUDO /usr/sbin/iotop --kilobytes --only --batch --time --delay=2 --processes --quiet | $SUDO tee --append /var/log/iotop.log > /dev/null &

fi

if [ "${MANAGE_PUPPET_MODULES}" = true ]; then

$SUDO ./install_modules.sh

fi