graphtimer v2 - Utility to plot timings

Question

GitHub repo (MIT)
Clone the repo and replace the contents of example.py with the one at the bottom and you'll have everything setup.

Explanation of the code

A long while ago I posted the question "Abstract graphing-and-timing functions". It was a god class, wasn't extendable/configurable, you also had to use strings to interact with timeit. And it just wasn't clear what was what.

A little while ago I posted an answer where I used a couple ikky timeit.timeit calls rather than build a graph. And Graipher showed me a nice looking graph. I've continued to see these nice graphs, with no easy way to make them myself. And so I decided I needed to way-too-early-spring clean my code.

1. (timer.py) A Timer object should be in charge of building and calling timeit.
   This object was based of timeit.Timer, however I thought it needed some additional functionality:

   1. It should be able to time multiple functions.
   2. It should be able to test against multiple values.
   3. It should only perform timings.

2. (graph.py) A graph class should be introduced that plots the data in the correct way. If you want to plot in something that isn't matplotlib then you just need to change the class.

3. (plotter.py) Statistical analysis should not be in either the Timer or the graph class. I also kept the analysis to be pretty basic as per the Python docs suggestion.
4. (plotter.py) Performing all the above should be simple with good defaults. And so Plotter handles interacting with everything. Whilst it's a small wrapper it means that I, and other, don't have to write that many lines to get the wanted graph.

What I'd like out of a review.

1. The thing I find most important right now is, is the design good?

   Is Plotter a poor design pattern? Is splitting the code out like I have done a bad idea?

2. Is the usage for a user clean and readable?

3. Is there a way to make my code more readable? I find MatPlotLib to be a bit on the not so clean side.
4. Any and all critiques are welcome.

Code

(I have left out some code, and the .pyi files. These are available on GitHub.)

graph.py

from .graphtimer import CATEGORY10
class MatPlotLib:
 def _graph_times(self, graph, data, domain, colors, error, fmt):
 for results, color in zip(data, colors):
 values = [v.value for v in results]
 if error:
 errors = zip(*[v.errors or [] for v in results])
 for error in errors:
 lower, upper = zip(*error)
 graph.fill_between(domain, upper, lower, facecolor=color, edgecolor=None, alpha=0.1)
 yield graph.plot(domain, values, fmt, color=color)[0]
 def graph(self, graph, data, domain, *, functions=None, colors=CATEGORY10, title=None, legend=True, error=True,
 x_label='Input', y_label='Time [s]', fmt='-'):
 lines = list(self._graph_times(graph, data, domain, colors, error, fmt))
 if x_label is not None:
 graph.set_xlabel(x_label)
 if y_label is not None:
 graph.set_ylabel(y_label)
 if title is not None and hasattr(graph, 'set_title'):
 graph.set_title(title)
 if legend and functions is not None and hasattr(graph, 'legend'):
 graph.legend(lines, [fn.__name__ for fn in functions], loc=0)
 return lines

timers.py

import timeit
SENTINAL = object()
class MultiTimer:
 """Interface to timeit.Timer to ease timing over multiple functions."""
 def __init__(self, functions, timer=timeit.Timer):
 self.timer = timer
 self.functions = functions
 def build_timer(self, fn, domain, stmt='fn(*args)', setup='pass', timer=SENTINAL, globals=SENTINAL,
 args_conv=SENTINAL):
 """Build a timeit.Timer"""
 if not isinstance(domain, tuple):
 domain = domain,
 if args_conv is not SENTINAL:
 domain = args_conv(*domain)
 if not isinstance(domain, tuple):
 domain = domain,
 if globals is SENTINAL:
 globals = {}
 else:
 globals = globals.copy()
 globals.update({'fn': fn, 'args': domain})
 # print(f'{self.timer}({stmt!r}, {setup!r}, {timer!r}, {globals!r})')
 if timer is SENTINAL:
 timer = timeit.default_timer
 return self.timer(stmt, setup, timer, globals=globals)
 def build_timers(self, domain, *args, **kwargs):
 """Build multiple timers from various inputs and functions"""
 return [
 [
 self.build_timer(fn, dom, *args, **kwargs)
 for fn in self.functions
 ]
 for dom in domain
 ]
 def _call(self, domain, repeat, call, *args, **kwargs):
 """Helper function to generate timing data."""
 if len(domain) == 0:
 raise ValueError('domain must have at least one argument.')
 functions = self.build_timers(domain, *args, **kwargs)
 output = [[[] for _ in domain] for _ in functions[0]]
 for _ in range(repeat):
 for j, fns in enumerate(functions):
 for i, fn in enumerate(fns):
 output[i][j].append(call(fn))
 return output
 def repeat(self, domain, repeat, number, *args, **kwargs):
 """Interface to timeit.Timer.repeat. `domain` is the values to pass to the functions."""
 return self._call(domain, repeat, lambda f: f.timeit(number), *args, **kwargs)
 def timeit(self, domain, number, *args, **kwargs):
 """Interface to timeit.Timer.timeit. `domain` is the values to pass to the functions."""
 return [
 [value[0] for value in values]
 for values in self.repeat(domain, 1, number, *args, **kwargs)
 ]
 def autorange(self, domain, *args, **kwargs):
 """Interface to timeit.Timer.autorange. `domain` is the values to pass to the functions."""
 return [
 [value[0] for value in values]
 for values in self._call(domain, 1, lambda f: f.autorange(), *args, **kwargs)
 ]
class TimerNamespaceMeta(type):
 """Convenience class to ease creation of a MultiTimer."""
 def __new__(mcs, name, bases, attrs):
 if 'functions' in attrs:
 raise TypeError('FunctionTimers cannot define `functions`')
 if 'multi_timer' in attrs:
 raise TypeError('FunctionTimers cannot define `multi_timer`')
 ret: TimerNamespace = super().__new__(mcs, name, bases, attrs)
 functions = [v for k, v in attrs.items() if k.startswith('test')]
 ret.functions = functions
 ret.multi_timer = ret.MULTI_TIMER(functions, ret.TIMER)
 return ret
class TimerNamespace(metaclass=TimerNamespaceMeta):
 """Convenience class to ease creation of a MultiTimer."""
 TIMER = timeit.Timer
 MULTI_TIMER = MultiTimer

plotter.py

from .graph import MatPlotLib
class Plotter:
 """Interface to the timer object. Returns objects made to ease usage."""
 def __init__(self, timer):
 self.timer = getattr(timer, 'multi_timer', timer)
 def timeit(self, number, domain, *args, **kwargs):
 """Interface to self.timer.timeit. Returns a PlotValues."""
 return self.repeat(1, number, domain, *args, **kwargs).min(errors=None)
 def repeat(self, repeat, number, domain, *args, **kwargs):
 """Interface to self.timer.repeat. Returns a PlotTimings."""
 return PlotTimings(
 self.timer.repeat(domain, repeat, number, *args, **kwargs),
 {
 'functions': self.timer.functions,
 'domain': domain
 }
 )
class _DataSet:
 """Holds timeit values and defines statistical methods around them."""
 def __init__(self, values):
 self.values = sorted(values)
 def quartile_indexes(self, outlier):
 """Generates the quartile indexes. Uses tukey's fences to remove outliers."""
 delta = (len(self.values) - 1) / 4
 quartiles = [int(round(delta * i)) for i in range(5)]
 if outlier is not None:
 if outlier < 0:
 raise ValueError("outlier should be non-negative.")
 iqr = outlier * (self.values[quartiles[3]] - self.values[quartiles[1]])
 low = self.values[quartiles[1]] - iqr
 high = self.values[quartiles[3]] + iqr
 for i, v in enumerate(self.values):
 if v >= low:
 quartiles[0] = i
 break
 for i, v in reversed(list(enumerate(self.values))):
 if v <= high:
 quartiles[4] = i
 break
 return tuple(quartiles)
 def errors(self, errors, outlier):
 """Returns tuples containing the quartiles wanted."""
 if errors is None:
 return None
 quartiles = self.quartile_indexes(outlier)
 # Allow out of quartile error bars using -1 and 5.
 quartiles += (-1, 0)
 return [
 (
 self.values[quartiles[start]],
 self.values[quartiles[stop]]
 )
 for start, stop in errors
 ]
 def quartile(self, quartile, outlier):
 """Return the value of the quartile provided."""
 quartiles = self.quartile_indexes(outlier)
 return self.values[quartiles[quartile]]
 def mean(self, start, end, outlier):
 """Return the mean of the values over the quartiles specified."""
 quartiles = self.quartile_indexes(outlier)
 start = quartiles[start]
 end = quartiles[end]
 return sum(self.values[start:end + 1]) / (1 + end - start)
class PlotTimings:
 """Thin interface over _DataSet"""
 def __init__(self, data, kwargs):
 self.data = [
 [_DataSet(results) for results in function_values]
 for function_values in data
 ]
 self.kwargs = kwargs
 def quartile(self, quartile, *, errors=None, outlier=1.5):
 """Interface to _DataSet.quartile and errors. Returns a PlotValues."""
 return PlotValues(
 [
 [
 _DataValues(
 ds.quartile(quartile, outlier),
 ds.errors(errors, outlier)
 )
 for ds in function_values
 ]
 for function_values in self.data
 ],
 self.kwargs
 )
 def min(self, *, errors=((-1, 3),), outlier=1.5):
 """Return the Q1 value and show the error from Q-1 Q3."""
 return self.quartile(0, errors=errors, outlier=outlier)
 def max(self, *, errors=((1, 5),), outlier=1.5):
 """Return the Q4 value and show the error from Q1 Q5."""
 return self.quartile(4, errors=errors, outlier=outlier)
 def mean(self, start=0, end=4, *, errors=((1, 3),), outlier=1.5):
 """Interface to _DataSet.mean and errors. Returns a PlotValues."""
 return PlotValues(
 [
 [
 _DataValues(
 ds.mean(start, end, outlier),
 ds.errors(errors, outlier)
 )
 for ds in function_values
 ]
 for function_values in self.data
 ],
 self.kwargs
 )
class _DataValues:
 """Holds the wanted statistical data from the timings."""
 def __init__(self, value, errors):
 self.value = value
 self.errors = errors
class PlotValues:
 """Thin interface to Graph.graph."""
 def __init__(self, data, kwargs):
 self.data = data
 self.kwargs = kwargs
 def plot(self, graph, graph_lib=MatPlotLib, **kwargs):
 g = graph_lib()
 return g.graph(
 graph,
 self.data,
 self.kwargs.pop('domain'),
 functions=self.kwargs.pop('functions'),
 **kwargs
 )

Example usage

I've included the same graph as I did on my old code. And two of Graipher's graphs.

1. Abstract graphing-and-timing functions - This is to ensure usage is simple in abnormal usage.
   It also shows that you can plot multiple error areas, highlighted in the unoptimised graph.

2. Plot timings for a range of inputs - This is to make sure standard usage is simple.

3. String reversal in Python - This is so I know logerithmic graphs display correctly.

   I'm running on Windows and don't have a C compiler, and so I can't include the additional two functions. However I think it nicely shows why the Python docs say to use min.

import time
import math
import matplotlib.pyplot as plt
import numpy as np
from graphtimer import flat, Plotter, TimerNamespace
class UnoptimisedRange(object):
 def __init__(self, size):
 self.size = size
 def __getitem__(self, i):
 if i >= self.size:
 raise IndexError()
 return i
class Peilonrayz(TimerNamespace):
 def test_comprehension(iterable):
 return [i for i in iterable]
 def test_append(iterable):
 a = []
 append = a.append
 for i in iterable:
 append(i)
 return a
SCALE = 10.
class Graipher(TimerNamespace):
 def test_o_n(n):
 time.sleep(n / SCALE)
 def test_o_n2(n):
 time.sleep(n ** 2 / SCALE)
 def test_o_log(n):
 time.sleep(math.log(n + 1) / SCALE)
 def test_o_exp(n):
 time.sleep((math.exp(n) - 1) / SCALE)
 def test_o_nlog(n):
 time.sleep(n * math.log(n + 1) / SCALE)
class Reverse(TimerNamespace):
 def test_orig(stri):
 output = ''
 length = len(stri)
 while length > 0:
 output += stri[-1]
 stri, length = (stri[0:length - 1], length - 1)
 return output
 def test_g(s):
 return s[::-1]
 def test_s(s):
 return ''.join(reversed(s))
def main():
 # Reverse
 fig, axs = plt.subplots()
 axs.set_yscale('log')
 axs.set_xscale('log')
 (
 Plotter(Reverse)
 .repeat(10, 10, np.logspace(0, 5), args_conv=lambda i: ' '*int(i))
 .min()
 .plot(axs, title='Reverse', fmt='-o')
 )
 fig.show()
 # Graipher
 fig, axs = plt.subplots()
 (
 Plotter(Graipher)
 .repeat(2, 1, [i / 10 for i in range(10)])
 .min()
 .plot(axs, title='Graipher', fmt='-o')
 )
 fig.show()
 # Peilonrayz
 fig, axs = plt.subplots(nrows=2, ncols=2, sharex=True, sharey=True)
 p = Plotter(Peilonrayz)
 axis = [
 ('Range', {'args_conv': range}),
 ('List', {'args_conv': lambda i: list(range(i))}),
 ('Unoptimised', {'args_conv': UnoptimisedRange}),
 ]
 for graph, (title, kwargs) in zip(iter(flat(axs)), axis):
 (
 p.repeat(100, 5, list(range(0, 10001, 1000)), **kwargs)
 .min(errors=((-1, 3), (-1, 4)))
 .plot(graph, title=title)
 )
 fig.show()
if __name__ == '__main__':
 main()

My graphs Graipher's graphs Reverse graphs