Long running Python "server status" collector

Question

I wrote a small program that collects the server status, caches it for a certain amount of seconds, and then sends that to a mothership server. It's broken down into a few pieces, starting with main.py, config.json, and a few small 'libs' to keep things visually separate:

main.py

#!/usr/bin/python
import os
import sys
import json
import time
import sched
import socket
import psutil
from lib import cpu, memory, disks, network, system, transport
__cache = []
__cache_timer = 0
__cache_keeper = 0
def main(scheduler, config, sock, hostname, callers):
 global __cache
 global __cache_timer
 global __cache_keeper
 payload = {
 "_id": {
 "time": time.time(),
 "id": config['identification']['id'],
 "hostname": hostname,
 "type": config['identification']['type']
 },
 "cpu": callers['cpu'].snapshot(),
 "memory": callers['memory'].snapshot(),
 "disks": callers['disks'].snapshot(),
 "network": callers['network'].snapshot(),
 "system": callers['system'].snapshot()
 }
 __cache.append(payload)
 if __cache_keeper < __cache_timer:
 __cache_keeper += config['interval']
 else:
 transport.Transport({"payload": json.dumps(__cache)}, config, sock)
 __cache_keeper = 0
 __cache = []
 # Schedule a new run at the specified interval
 scheduler.enter(config['interval'], 1, main, (scheduler, config, sock, hostname, callers))
 scheduler.run()
if __name__ == '__main__':
 try:
 config = (json.loads(open(os.path.dirname(os.path.abspath(__file__)) + "/config.json").read()))['config']
 config['identification']['type'] = config['identification'].get('type', 'false')
 config['disable_cache'] = False
 if config['cache'].get('enabled') is True:
 __cache_timer = config['cache'].get('time_seconds_to_cache_between_sends', 60)
 config['interval'] = config['cache'].get('interval_seconds_between_captures', 5)
 # If the interval is higher, just exit
 if config['interval'] > __cache_timer:
 print >> sys.stderr, "Report interval is higher than cache timer."
 sys.exit(1)
 sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
 scheduler = sched.scheduler(time.time, time.sleep)
 hostname = config['identification'].get('hostname', socket.gethostname())
 callers = {
 "cpu": cpu.CPU(psutil),
 "memory": memory.Memory(psutil),
 "disks": disks.Disks(psutil),
 "network": network.Network(psutil),
 "system": system.System(psutil)
 }
 main(scheduler, config, sock, hostname, callers)
 except KeyboardInterrupt:
 print >> sys.stderr, '\nExiting by user request.\n'
 sys.exit(0)
 except Exception as e:
 location = '\n' + type(e).__name__
 print >> sys.stderr, location, '=>', str(e)
 sys.exit(1)

config.json

{
 "config": {
 "mothership": {
 "host": "127.0.0.1",
 "port": 1336
 },
 "cache": {
 "enabled": true,
 "time_seconds_to_cache_between_sends": 5,
 "interval_seconds_between_captures": 1
 },
 "identification": {
 "id": "some-id-here"
 }
 }
}

lib/cpu.py

from subprocess import check_output
class CPU:
 psutil = None
 cpu_count = {}
 cpu_passthrough = 0
 def __init__(self, psutil):
 self.psutil = psutil
 def snapshot(self):
 """
 Generate a snapshot of the current CPU state
 """
 cpu_time = self.psutil.cpu_times()
 # Only update the CPU counts every 100th pass through
 if self.cpu_count == {} or self.cpu_passthrough % 100 == 0:
 self.cpu_count = {
 "virtual": self.psutil.cpu_count(),
 "physical": self.psutil.cpu_count(logical=False)
 }
 return {
 "cpu_percent": self.psutil.cpu_percent(interval=1, percpu=True),
 "cpu_times": {
 "user": cpu_time[0],
 "system": cpu_time[1],
 "idle": cpu_time[2]
 },
 "cpu_count": self.cpu_count,
 "load_average": check_output(["cat", "/proc/loadavg"])
 }

lib/disks.py

class Disks:
 psutil = None
 disk_usage = None
 disk_partitions = None
 disk_passthrough = 0
 def __init__(self, psutil):
 self.psutil = psutil
 def snapshot(self):
 """
 Generate a snapshot of the current disk state
 """
 # Only grab the disk partitions every 25th pass
 if self.disk_partitions is None or self.disk_passthrough % 25 == 0:
 self.disk_partitions = self.psutil.disk_partitions(all=True)
 # Only grab the disk usage every 5th pass
 if self.disk_usage is None or self.disk_passthrough % 5 == 0:
 self.disk_usage = self.psutil.disk_usage('/')
 self.disk_passthrough += 1
 return {
 "disk_usage": self.disk_usage,
 "disk_partitions": self.disk_partitions,
 "disk_io_counters": self.psutil.disk_io_counters(perdisk=True)
 }

lib/memory.py

class Memory:
 psutil = None
 def __init__(self, psutil):
 self.psutil = psutil
 def snapshot(self):
 """
 Generate a snapshot of the current memory state
 """
 return {
 "virtual": self.psutil.virtual_memory(),
 "swap": self.psutil.swap_memory()
 }

lib/network.py

class Network:
 psutil = None
 def __init__(self, psutil):
 self.psutil = psutil
 def snapshot(self):
 """
 Generate a snapshot of the current network state
 """
 return {
 "net_io_counters": self.psutil.net_io_counters(pernic=True)
 }

lib/system.py

class System:
 psutil = None
 def __init__(self, psutil):
 self.psutil = psutil
 def snapshot(self):
 """
 Generate a snapshot of the current system state
 """
 return {
 "users": self.psutil.users(),
 "boot_time": self.psutil.boot_time()
 }

lib/transport.py

class Transport():
 def __init__(self, payload, config, sock):
 payload = str(payload)
 sock.setblocking(0)
 sock.sendto(payload, (config.get('mothership').get('host'), 
 config.get('mothership').get('port')))

Have I done anything wrong? What could I improve?

• 200_success: Are you deliberately reinventing the wheel and using a custom protocol?
• jsanc623: Reinventing the wheel yes - educationally (I use nagios and have rolled munin out for a client). As for the custom protocol - I don't think so? Just sending some JSON out via UDP.

Answer 1

Don't prefix names with two underscores; those are reserved for name mangling. Use a single underscore.

When opening files, use with:

with open(...) as my_file:
 ... # use my_file

This deals with cleanup.

Don't do this:

class MyClass:
 var = ...

The var here is actually a global variable. This is one of Python's less obvious gotcha's. Instead, assign it in the constructor.

I would be very wary of initializing data to wrong-but-valid-looking data. If possible, don't initialize early. If not, make it explicit - normally this means to use None.

• jsanc623: When you say var = ..., I'm assuming you mean any variables assigned (say x = ...)? Did not know that they instantly became global variables - just thought they became properties of the class (like every other sane OOP implementation). Will have to look into this more.
• jsanc623: If you meant this (i.imgur.com/PWrH41E.png) - then yes, it's intentionally global
• Veedrac: @jsanc623 I'm referring to any assigned variables, yes. Class variables are globals; they're just namespaced. You should use instance variables - in fact you end up shadowing most of these with instance variables anyway. Python's OOP is really straightforward once you forget everything that Java (or similar) taught you: ClassName.x is a class variable, instance_name.x is an instance variable, things inside the class ClassName: body are run when the class object is instantiated, things inside __init__ are run when the instance is instantiated.
• jsanc623: Very interesting!