[Python-checkins] benchmarks: Improve the precision and reproduceability of benchmark results:

antoine.pitrou python-checkins at python.org
Sat Oct 25 22:17:34 CEST 2014

• Previous message: [Python-checkins] cpython (3.4): Issue #22711: improve links in new distribution docs
• Next message: [Python-checkins] cpython: Issue #22261: Add a note to PCbuild\readme.txt about MSBuild switches.
• Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

---

https://hg.python.org/benchmarks/rev/dc7d29be5a9e
changeset: 217:dc7d29be5a9e
user: Antoine Pitrou <solipsis at pitrou.net>
date: Sat Oct 25 22:17:25 2014 +0200
summary:
 Improve the precision and reproduceability of benchmark results:
- use time.perf_counter() if both interpreters support it
- force deterministic hash() results
files:
 perf.py | 37 ++++++++++++++-
 performance/bm_call_method.py | 8 +-
 performance/bm_call_method_slots.py | 8 +-
 performance/bm_call_method_unknown.py | 8 +-
 performance/bm_call_simple.py | 8 +-
 performance/bm_chameleon.py | 7 +-
 performance/bm_chaos.py | 10 ++--
 performance/bm_django.py | 6 +-
 performance/bm_django_v2.py | 6 +-
 performance/bm_elementtree.py | 6 +-
 performance/bm_fannkuch.py | 6 +-
 performance/bm_float.py | 6 +-
 performance/bm_go.py | 6 +-
 performance/bm_hexiom2.py | 6 +-
 performance/bm_html5lib.py | 6 +-
 performance/bm_json.py | 12 ++--
 performance/bm_json_v2.py | 6 +-
 performance/bm_logging.py | 18 +++---
 performance/bm_mako.py | 6 +-
 performance/bm_mako_v2.py | 6 +-
 performance/bm_meteor_contest.py | 6 +-
 performance/bm_nbody.py | 6 +-
 performance/bm_nqueens.py | 6 +-
 performance/bm_pathlib.py | 8 +-
 performance/bm_pickle.py | 30 ++++++------
 performance/bm_pidigits.py | 6 +-
 performance/bm_raytrace.py | 7 +-
 performance/bm_regex_compile.py | 12 ++--
 performance/bm_regex_effbot.py | 6 +-
 performance/bm_regex_v8.py | 6 +-
 performance/bm_richards.py | 6 +-
 performance/bm_rietveld.py | 6 +-
 performance/bm_spambayes.py | 6 +-
 performance/bm_spectral_norm.py | 6 +-
 performance/bm_spitfire.py | 10 ++--
 performance/bm_telco.py | 13 ++--
 performance/bm_threading.py | 12 ++--
 performance/bm_tornado_http.py | 6 +-
 performance/bm_unpack_sequence.py | 20 ++++----
 performance/util.py | 8 ++-
 40 files changed, 202 insertions(+), 166 deletions(-)
diff --git a/perf.py b/perf.py
--- a/perf.py
+++ b/perf.py
@@ -120,6 +120,29 @@
 return 'lib'
 
 
+def supported_timers(python):
+ """Return a list of supported timers by the given Python interpreter,
+ in decreasing order of priority.
+ """
+ version = interpreter_version(python)
+ if version >= '3.3':
+ return ['perf_counter', 'time']
+ else:
+ return ['time']
+
+
+def choose_timer(base_python, changed_python):
+ """Choose the best timer supported by both *base_python* and
+ *changed_python*.
+ """
+ u = supported_timers(base_python)
+ v = set(supported_timers(changed_python))
+ for timer in u:
+ if timer in v:
+ return timer
+ assert 0 # At least time.time() should always be available
+
+
 def avg(seq):
 return sum(seq) / float(len(seq))
 
@@ -969,6 +992,9 @@
 for k in ("COMSPEC", "SystemRoot"):
 if k in os.environ and k not in fixed_env:
 fixed_env[k] = os.environ[k]
+ # Make hashing deterministic (this may make some benchmarks more
+ # reproduceable).
+ fixed_env["PYTHONHASHSEED"] = "1"
 return fixed_env
 
 
@@ -1130,7 +1156,8 @@
 trials = max(1, int(trials * iteration_scaling))
 
 RemovePycs()
- command = python + [bm_path, "-n", trials] + extra_args
+ bench_args = [bm_path, "-n", trials, "--timer", options.timer]
+ command = python + bench_args + extra_args
 output = CallAndCaptureOutput(command, bm_env,
 track_memory=options.track_memory,
 inherit_env=options.inherit_env)
@@ -2401,6 +2428,7 @@
 usage="%prog [options] baseline_python changed_python",
 description=("Compares the performance of baseline_python with" +
 " changed_python and prints a report."))
+
 parser.add_option("-r", "--rigorous", action="store_true",
 help=("Spend longer running tests to get more" +
 " accurate results"))
@@ -2410,6 +2438,9 @@
 help="Print more output")
 parser.add_option("-m", "--track_memory", action="store_true",
 help="Track memory usage. This only works on Linux.")
+ parser.add_option("--timer", action="store",
+ help="Override timer function.")
+
 parser.add_option("-a", "--args", default="",
 help=("Pass extra arguments to the python binaries."
 " If there is a comma in this option's value, the"
@@ -2488,6 +2519,10 @@
 if options.diff_instrumentation:
 info("Suppressing performance data due to --diff_instrumentation")
 
+ if not options.timer:
+ options.timer = choose_timer(base_cmd_prefix, changed_cmd_prefix)
+ info("Automatically selected timer: %s", options.timer)
+
 should_run = ParseBenchmarksOption(options.benchmarks, bench_groups,
 options.fast)
 
diff --git a/performance/bm_call_method.py b/performance/bm_call_method.py
--- a/performance/bm_call_method.py
+++ b/performance/bm_call_method.py
@@ -113,11 +113,11 @@
 pass
 
 
-def test_calls(iterations):
+def test_calls(iterations, timer):
 times = []
 f = Foo()
 for _ in xrange(iterations):
- t0 = time.time()
+ t0 = timer()
 # 20 calls
 f.foo(1, 2, 3, 4)
 f.foo(1, 2, 3, 4)
@@ -139,7 +139,7 @@
 f.foo(1, 2, 3, 4)
 f.foo(1, 2, 3, 4)
 f.foo(1, 2, 3, 4)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
@@ -153,6 +153,6 @@
 options, _ = parser.parse_args()
 
 # Priming run.
- test_calls(1)
+ test_calls(1, time.time)
 
 util.run_benchmark(options, options.num_runs, test_calls)
diff --git a/performance/bm_call_method_slots.py b/performance/bm_call_method_slots.py
--- a/performance/bm_call_method_slots.py
+++ b/performance/bm_call_method_slots.py
@@ -117,13 +117,13 @@
 pass
 
 
-def test_calls(iterations):
+def test_calls(iterations, timer):
 times = []
 f = Foo()
 if hasattr(f, '__dict__'):
 raise Exception("f has a __dict__ attribute!")
 for _ in xrange(iterations):
- t0 = time.time()
+ t0 = timer()
 # 20 calls
 f.foo(1, 2, 3, 4)
 f.foo(1, 2, 3, 4)
@@ -145,7 +145,7 @@
 f.foo(1, 2, 3, 4)
 f.foo(1, 2, 3, 4)
 f.foo(1, 2, 3, 4)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
@@ -159,6 +159,6 @@
 options, _ = parser.parse_args()
 
 # Priming run.
- test_calls(1)
+ test_calls(1, time.time)
 
 util.run_benchmark(options, options.num_runs, test_calls)
diff --git a/performance/bm_call_method_unknown.py b/performance/bm_call_method_unknown.py
--- a/performance/bm_call_method_unknown.py
+++ b/performance/bm_call_method_unknown.py
@@ -313,13 +313,13 @@
 pass
 
 
-def test_calls(iterations):
+def test_calls(iterations, timer):
 times = []
 a = Foo()
 b = Bar()
 c = Baz()
 for _ in xrange(iterations):
- t0 = time.time()
+ t0 = timer()
 # 18 calls
 a.foo(b, c)
 b.foo(c, a)
@@ -339,7 +339,7 @@
 a.foo(b, c)
 b.foo(c, a)
 c.foo(a, b)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
@@ -353,6 +353,6 @@
 options, _ = parser.parse_args()
 
 # Priming run.
- test_calls(1)
+ test_calls(1, time.time)
 
 util.run_benchmark(options, options.num_runs, test_calls)
diff --git a/performance/bm_call_simple.py b/performance/bm_call_simple.py
--- a/performance/bm_call_simple.py
+++ b/performance/bm_call_simple.py
@@ -115,10 +115,10 @@
 pass
 
 
-def test_calls(iterations):
+def test_calls(iterations, timer):
 times = []
 for _ in xrange(iterations):
- t0 = time.time()
+ t0 = timer()
 # 20 calls
 foo(1, 2, 3, 4)
 foo(1, 2, 3, 4)
@@ -140,7 +140,7 @@
 foo(1, 2, 3, 4)
 foo(1, 2, 3, 4)
 foo(1, 2, 3, 4)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
@@ -154,6 +154,6 @@
 options, _ = parser.parse_args()
 
 # Priming run.
- test_calls(1)
+ test_calls(1, time.time)
 
 util.run_benchmark(options, options.num_runs, test_calls)
diff --git a/performance/bm_chameleon.py b/performance/bm_chameleon.py
--- a/performance/bm_chameleon.py
+++ b/performance/bm_chameleon.py
@@ -15,16 +15,15 @@
 </tr>
 </table>""" % compat.unicode.__name__
 
-def main(n):
+def main(n, timer):
 tmpl = PageTemplate(BIGTABLE_ZPT)
 options = {'table': [dict(a=1, b=2, c=3, d=4, e=5, f=6, g=7, h=8, i=9, j=10)
 for x in range(1000)]}
- import time
 l = []
 for k in range(n):
- t0 = time.time()
+ t0 = timer()
 tmpl(options=options)
- l.append(time.time() - t0)
+ l.append(timer() - t0)
 return l
 
 if __name__ == '__main__':
diff --git a/performance/bm_chaos.py b/performance/bm_chaos.py
--- a/performance/bm_chaos.py
+++ b/performance/bm_chaos.py
@@ -199,14 +199,14 @@
 if point.y < self.miny:
 point.y = self.miny
 
- def create_image_chaos(self, w, h, n):
+ def create_image_chaos(self, timer, w, h, n):
 im = [[1] * h for i in range(w)]
 point = GVector((self.maxx + self.minx) / 2,
 (self.maxy + self.miny) / 2, 0)
 colored = 0
 times = []
 for _ in range(n):
- t1 = time.time()
+ t1 = timer()
 for i in xrange(5000):
 point = self.transform_point(point)
 x = (point.x - self.minx) / self.width * w
@@ -218,12 +218,12 @@
 if y == h:
 y -= 1
 im[x][h - y - 1] = 0
- t2 = time.time()
+ t2 = timer()
 times.append(t2 - t1)
 return times
 
 
-def main(n):
+def main(n, timer):
 splines = [
 Spline([
 GVector(1.597350, 3.304460, 0.000000),
@@ -248,7 +248,7 @@
 3, [0, 0, 0, 1, 1, 1])
 ]
 c = Chaosgame(splines, 0.25)
- return c.create_image_chaos(1000, 1200, n)
+ return c.create_image_chaos(timer, 1000, 1200, n)
 
 
 
diff --git a/performance/bm_django.py b/performance/bm_django.py
--- a/performance/bm_django.py
+++ b/performance/bm_django.py
@@ -32,7 +32,7 @@
 </table>
 """)
 
-def test_django(count):
+def test_django(count, timer):
 table = [xrange(150) for _ in xrange(150)]
 context = Context({"table": table})
 
@@ -42,9 +42,9 @@
 
 times = []
 for _ in xrange(count):
- t0 = time.time()
+ t0 = timer()
 data = DJANGO_TMPL.render(context)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
diff --git a/performance/bm_django_v2.py b/performance/bm_django_v2.py
--- a/performance/bm_django_v2.py
+++ b/performance/bm_django_v2.py
@@ -28,7 +28,7 @@
 </table>
 """)
 
-def test_django(count):
+def test_django(count, timer):
 table = [xrange(150) for _ in xrange(150)]
 context = Context({"table": table})
 
@@ -38,9 +38,9 @@
 
 times = []
 for _ in xrange(count):
- t0 = time.time()
+ t0 = timer()
 data = DJANGO_TMPL.render(context)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
diff --git a/performance/bm_elementtree.py b/performance/bm_elementtree.py
--- a/performance/bm_elementtree.py
+++ b/performance/bm_elementtree.py
@@ -179,7 +179,7 @@
 raise RuntimeError("unexpected output detected")
 
 
-def run_etree_benchmark(iterations, etree, bench_func):
+def run_etree_benchmark(iterations, timer, etree, bench_func):
 times = []
 
 xml_root = build_xml_tree(etree)
@@ -195,9 +195,9 @@
 bench_func(etree, file_path, xml_data, xml_root)
 
 for _ in xrange(iterations):
- t0 = time.time()
+ t0 = timer()
 bench_func(etree, file_path, xml_data, xml_root)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 finally:
 try:
diff --git a/performance/bm_fannkuch.py b/performance/bm_fannkuch.py
--- a/performance/bm_fannkuch.py
+++ b/performance/bm_fannkuch.py
@@ -54,12 +54,12 @@
 
 DEFAULT_ARG = 9
 
-def main(n):
+def main(n, timer):
 times = []
 for i in xrange(n):
- t0 = time.time()
+ t0 = timer()
 fannkuch(DEFAULT_ARG)
- tk = time.time()
+ tk = timer()
 times.append(tk - t0)
 return times
 
diff --git a/performance/bm_float.py b/performance/bm_float.py
--- a/performance/bm_float.py
+++ b/performance/bm_float.py
@@ -47,14 +47,14 @@
 
 POINTS = 100000
 
-def main(arg):
+def main(arg, timer):
 # XXX warmup
 
 times = []
 for i in xrange(arg):
- t0 = time.time()
+ t0 = timer()
 o = benchmark(POINTS)
- tk = time.time()
+ tk = timer()
 times.append(tk - t0)
 return times
 
diff --git a/performance/bm_go.py b/performance/bm_go.py
--- a/performance/bm_go.py
+++ b/performance/bm_go.py
@@ -424,14 +424,14 @@
 board = Board()
 pos = computer_move(board)
 
-def main(n):
+def main(n, timer):
 times = []
 for i in range(5):
 versus_cpu() # warmup
 for i in range(n):
- t1 = time.time()
+ t1 = timer()
 versus_cpu()
- t2 = time.time()
+ t2 = timer()
 times.append(t2 - t1)
 return times
 
diff --git a/performance/bm_hexiom2.py b/performance/bm_hexiom2.py
--- a/performance/bm_hexiom2.py
+++ b/performance/bm_hexiom2.py
@@ -518,14 +518,14 @@
 if output.getvalue() != expected:
 raise AssertionError("got a wrong answer:\n%s" % output.getvalue())
 
-def main(n):
+def main(n, timer):
 # only run 1/25th of the requested number of iterations.
 # with the default n=50 from runner.py, this means twice.
 l = []
 for i in xrange(n):
- t0 = time.time()
+ t0 = timer()
 run_level36()
- time_elapsed = time.time() - t0
+ time_elapsed = timer() - t0
 l.append(time_elapsed)
 return l
 
diff --git a/performance/bm_html5lib.py b/performance/bm_html5lib.py
--- a/performance/bm_html5lib.py
+++ b/performance/bm_html5lib.py
@@ -23,16 +23,16 @@
 import html5lib
 
 
-def test_html5lib(count, spec_data):
+def test_html5lib(count, timer, spec_data):
 # No warm-up runs for this benchmark; in real life, the parser doesn't get
 # to warm up (this isn't a daemon process).
 
 times = []
 for _ in xrange(count):
 spec_data.seek(0)
- t0 = time.time()
+ t0 = timer()
 html5lib.parse(spec_data)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
diff --git a/performance/bm_json.py b/performance/bm_json.py
--- a/performance/bm_json.py
+++ b/performance/bm_json.py
@@ -76,7 +76,7 @@
 DICT_GROUP = [mutate_dict(DICT, random_source) for _ in range(3)]
 
 
-def test_json_dump(num_obj_copies, json, options):
+def test_json_dump(num_obj_copies, timer, json, options):
 # Warm-up runs.
 json.dumps(DICT)
 json.dumps(TUPLE)
@@ -85,7 +85,7 @@
 loops = num_obj_copies // 20 # We do 20 runs per loop.
 times = []
 for _ in xrange(options.num_runs):
- t0 = time.time()
+ t0 = timer()
 for _ in xrange(loops):
 json.dumps(DICT)
 json.dumps(DICT)
@@ -147,12 +147,12 @@
 json.dumps(DICT_GROUP)
 json.dumps(DICT_GROUP)
 json.dumps(DICT_GROUP)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
 
-def test_json_load(num_obj_copies, json, options):
+def test_json_load(num_obj_copies, timer, json, options):
 json_dict = json.dumps(DICT)
 json_tuple = json.dumps(TUPLE)
 json_dict_group = json.dumps(DICT_GROUP)
@@ -165,7 +165,7 @@
 loops = num_obj_copies // 20 # We do 20 runs per loop.
 times = []
 for _ in xrange(options.num_runs):
- t0 = time.time()
+ t0 = timer()
 for _ in xrange(loops):
 json.loads(json_dict)
 json.loads(json_dict)
@@ -227,7 +227,7 @@
 json.loads(json_dict_group)
 json.loads(json_dict_group)
 json.loads(json_dict_group)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
diff --git a/performance/bm_json_v2.py b/performance/bm_json_v2.py
--- a/performance/bm_json_v2.py
+++ b/performance/bm_json_v2.py
@@ -17,15 +17,15 @@
 
 cases = ['EMPTY', 'SIMPLE', 'NESTED', 'HUGE']
 
-def main(n):
+def main(n, timer):
 l = []
 for i in xrange(n):
- t0 = time.time()
+ t0 = timer()
 for case in cases:
 data, count = globals()[case]
 for i in xrange(count):
 json.dumps(data)
- l.append(time.time() - t0)
+ l.append(timer() - t0)
 return l
 
 if __name__ == '__main__':
diff --git a/performance/bm_logging.py b/performance/bm_logging.py
--- a/performance/bm_logging.py
+++ b/performance/bm_logging.py
@@ -19,11 +19,11 @@
 MESSAGE = 'some important information to be logged'
 
 
-def test_no_output(iterations, logger):
+def test_no_output(iterations, timer, logger):
 times = []
 m = MESSAGE
 for _ in xrange(iterations):
- t0 = time.time()
+ t0 = timer()
 for _ in xrange(10000):
 logger.debug(m)
 logger.debug(m)
@@ -35,16 +35,16 @@
 logger.debug(m)
 logger.debug(m)
 logger.debug(m)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
 
-def test_simple_output(iterations, logger):
+def test_simple_output(iterations, timer, logger):
 times = []
 m = MESSAGE
 for _ in xrange(iterations):
- t0 = time.time()
+ t0 = timer()
 for _ in xrange(1000):
 logger.warn(m)
 logger.warn(m)
@@ -56,17 +56,17 @@
 logger.warn(m)
 logger.warn(m)
 logger.warn(m)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
 
-def test_formatted_output(iterations, logger):
+def test_formatted_output(iterations, timer, logger):
 times = []
 f = FORMAT
 m = MESSAGE
 for _ in xrange(iterations):
- t0 = time.time()
+ t0 = timer()
 for _ in xrange(1000):
 logger.warn(f, m)
 logger.warn(f, m)
@@ -78,7 +78,7 @@
 logger.warn(f, m)
 logger.warn(f, m)
 logger.warn(f, m)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
diff --git a/performance/bm_mako.py b/performance/bm_mako.py
--- a/performance/bm_mako.py
+++ b/performance/bm_mako.py
@@ -25,7 +25,7 @@
 </table>
 """)
 
-def test_mako(count):
+def test_mako(count, timer):
 table = [xrange(150) for _ in xrange(150)]
 
 # Warm up Mako.
@@ -34,9 +34,9 @@
 
 times = []
 for _ in xrange(count):
- t0 = time.time()
+ t0 = timer()
 MAKO_TMPL.render(table = table)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
diff --git a/performance/bm_mako_v2.py b/performance/bm_mako_v2.py
--- a/performance/bm_mako_v2.py
+++ b/performance/bm_mako_v2.py
@@ -118,7 +118,7 @@
 """
 
 
-def test_mako(count):
+def test_mako(count, timer):
 
 lookup = TemplateLookup()
 lookup.put_string('base.mako', BASE_TEMPLATE)
@@ -132,11 +132,11 @@
 
 times = []
 for i in xrange(count):
- t0 = time.time()
+ t0 = timer()
 data = template.render(table=table, paragraphs=paragraphs,
 lorem=LOREM_IPSUM, title=title,
 img_count=50, xrange=xrange)
- t1 = time.time()
+ t1 = timer()
 times.append(t1-t0)
 return times
 
diff --git a/performance/bm_meteor_contest.py b/performance/bm_meteor_contest.py
--- a/performance/bm_meteor_contest.py
+++ b/performance/bm_meteor_contest.py
@@ -131,10 +131,10 @@
 
 SOLVE_ARG = 60
 
-def main(n):
+def main(n, timer):
 times = []
 for i in xrange(n):
- t0 = time.time()
+ t0 = timer()
 free = frozenset(xrange(len(board)))
 curr_board = [-1] * len(board)
 pieces_left = list(range(len(pieces)))
@@ -142,7 +142,7 @@
 solve(SOLVE_ARG, 0, free, curr_board, pieces_left, solutions)
 #print len(solutions), 'solutions found\n'
 #for i in (0, -1): print_board(solutions[i])
- tk = time.time()
+ tk = timer()
 times.append(tk - t0)
 return times
 
diff --git a/performance/bm_nbody.py b/performance/bm_nbody.py
--- a/performance/bm_nbody.py
+++ b/performance/bm_nbody.py
@@ -124,7 +124,7 @@
 v[2] = pz / m
 
 
-def test_nbody(iterations):
+def test_nbody(iterations, timer):
 # Warm-up runs.
 report_energy()
 advance(0.01, 20000)
@@ -132,11 +132,11 @@
 
 times = []
 for _ in xrange(iterations):
- t0 = time.time()
+ t0 = timer()
 report_energy()
 advance(0.01, 20000)
 report_energy()
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
diff --git a/performance/bm_nqueens.py b/performance/bm_nqueens.py
--- a/performance/bm_nqueens.py
+++ b/performance/bm_nqueens.py
@@ -60,16 +60,16 @@
 yield vec
 
 
-def test_n_queens(iterations):
+def test_n_queens(iterations, timer):
 # Warm-up runs.
 list(n_queens(8))
 list(n_queens(8))
 
 times = []
 for _ in xrange(iterations):
- t0 = time.time()
+ t0 = timer()
 list(n_queens(8))
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
diff --git a/performance/bm_pathlib.py b/performance/bm_pathlib.py
--- a/performance/bm_pathlib.py
+++ b/performance/bm_pathlib.py
@@ -43,7 +43,7 @@
 shutil.rmtree(TMP_PATH)
 
 
-def test_pathlib(count):
+def test_pathlib(count, timer):
 base_path = Path(TMP_PATH)
 
 # Warm up the filesystem cache and keep some objects in memory.
@@ -53,8 +53,8 @@
 assert len(path_objects) == NUM_FILES
 
 times = []
- for _ in xrange(count // 2):
- t0 = time.time()
+ for _ in xrange(max(1, count // 2)):
+ t0 = timer()
 # Do something simple with each path.
 for p in base_path:
 p.st_mtime
@@ -64,7 +64,7 @@
 p.st_mtime
 for p in base_path.glob("*.py"):
 p.st_mtime
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
diff --git a/performance/bm_pickle.py b/performance/bm_pickle.py
--- a/performance/bm_pickle.py
+++ b/performance/bm_pickle.py
@@ -86,7 +86,7 @@
 DICT_GROUP = [mutate_dict(DICT, random_source) for _ in range(3)]
 
 
-def test_pickle(num_obj_copies, pickle, options):
+def test_pickle(num_obj_copies, timer, pickle, options):
 # Warm-up runs.
 pickle.dumps(DICT, options.protocol)
 pickle.dumps(TUPLE, options.protocol)
@@ -95,7 +95,7 @@
 loops = num_obj_copies // 20 # We do 20 runs per loop.
 times = []
 for _ in xrange(options.num_runs):
- t0 = time.time()
+ t0 = timer()
 for _ in xrange(loops):
 pickle.dumps(DICT, options.protocol)
 pickle.dumps(DICT, options.protocol)
@@ -157,12 +157,12 @@
 pickle.dumps(DICT_GROUP, options.protocol)
 pickle.dumps(DICT_GROUP, options.protocol)
 pickle.dumps(DICT_GROUP, options.protocol)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
 
-def test_unpickle(num_obj_copies, pickle, options):
+def test_unpickle(num_obj_copies, timer, pickle, options):
 pickled_dict = pickle.dumps(DICT, options.protocol)
 pickled_tuple = pickle.dumps(TUPLE, options.protocol)
 pickled_dict_group = pickle.dumps(DICT_GROUP, options.protocol)
@@ -175,7 +175,7 @@
 loops = num_obj_copies // 20 # We do 20 runs per loop.
 times = []
 for _ in xrange(options.num_runs):
- t0 = time.time()
+ t0 = timer()
 for _ in xrange(loops):
 pickle.loads(pickled_dict)
 pickle.loads(pickled_dict)
@@ -237,7 +237,7 @@
 pickle.loads(pickled_dict_group)
 pickle.loads(pickled_dict_group)
 pickle.loads(pickled_dict_group)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
@@ -245,7 +245,7 @@
 LIST = [[list(range(10)), list(range(10))] for _ in xrange(10)]
 
 
-def test_pickle_list(loops, pickle, options):
+def test_pickle_list(loops, timer, pickle, options):
 # Warm-up runs.
 pickle.dumps(LIST, options.protocol)
 pickle.dumps(LIST, options.protocol)
@@ -253,7 +253,7 @@
 loops = loops // 5 # Scale to compensate for the workload.
 times = []
 for _ in xrange(options.num_runs):
- t0 = time.time()
+ t0 = timer()
 for _ in xrange(loops):
 pickle.dumps(LIST, options.protocol)
 pickle.dumps(LIST, options.protocol)
@@ -265,12 +265,12 @@
 pickle.dumps(LIST, options.protocol)
 pickle.dumps(LIST, options.protocol)
 pickle.dumps(LIST, options.protocol)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
 
-def test_unpickle_list(loops, pickle, options):
+def test_unpickle_list(loops, timer, pickle, options):
 pickled_list = pickle.dumps(LIST, options.protocol)
 
 # Warm-up runs.
@@ -280,7 +280,7 @@
 loops = loops // 5 # Scale to compensate for the workload.
 times = []
 for _ in xrange(options.num_runs):
- t0 = time.time()
+ t0 = timer()
 for _ in xrange(loops):
 pickle.loads(pickled_list)
 pickle.loads(pickled_list)
@@ -292,14 +292,14 @@
 pickle.loads(pickled_list)
 pickle.loads(pickled_list)
 pickle.loads(pickled_list)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
 
 MICRO_DICT = dict((key, dict.fromkeys(range(10))) for key in xrange(100))
 
-def test_pickle_dict(loops, pickle, options):
+def test_pickle_dict(loops, timer, pickle, options):
 # Warm-up runs.
 pickle.dumps(MICRO_DICT, options.protocol)
 pickle.dumps(MICRO_DICT, options.protocol)
@@ -307,14 +307,14 @@
 loops = max(1, loops // 10)
 times = []
 for _ in xrange(options.num_runs):
- t0 = time.time()
+ t0 = timer()
 for _ in xrange(loops):
 pickle.dumps(MICRO_DICT, options.protocol)
 pickle.dumps(MICRO_DICT, options.protocol)
 pickle.dumps(MICRO_DICT, options.protocol)
 pickle.dumps(MICRO_DICT, options.protocol)
 pickle.dumps(MICRO_DICT, options.protocol)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
diff --git a/performance/bm_pidigits.py b/performance/bm_pidigits.py
--- a/performance/bm_pidigits.py
+++ b/performance/bm_pidigits.py
@@ -16,7 +16,7 @@
 
 NDIGITS = 2000
 
-def test_pidgits(iterations):
+def test_pidgits(iterations, timer):
 _map = imap
 _count = itertools.count
 _islice = itertools.islice
@@ -57,9 +57,9 @@
 
 times = []
 for _ in xrange(iterations):
- t0 = time.time()
+ t0 = timer()
 calc_ndigits(NDIGITS)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
diff --git a/performance/bm_raytrace.py b/performance/bm_raytrace.py
--- a/performance/bm_raytrace.py
+++ b/performance/bm_raytrace.py
@@ -354,15 +354,14 @@
 s.addObject(Halfspace(Point(0,0,0), Vector.UP), CheckerboardSurface())
 s.render(c)
 
-def main(n):
- import time
+def main(n, timer):
 times = []
 for i in range(5):
 _main() # warmup
 for i in range(n):
- t1 = time.time()
+ t1 = timer()
 _main()
- t2 = time.time()
+ t2 = timer()
 times.append(t2 - t1)
 return times
 
diff --git a/performance/bm_regex_compile.py b/performance/bm_regex_compile.py
--- a/performance/bm_regex_compile.py
+++ b/performance/bm_regex_compile.py
@@ -41,10 +41,10 @@
 re.sub = capture_sub
 try:
 import bm_regex_effbot
- bm_regex_effbot.test_regex_effbot(1)
+ bm_regex_effbot.test_regex_effbot(1, time.time)
 
 import bm_regex_v8
- bm_regex_v8.test_regex_v8(1)
+ bm_regex_v8.test_regex_v8(1, time.time)
 finally:
 re.compile = real_compile
 re.search = real_search
@@ -52,7 +52,7 @@
 return regexes
 
 
-def test_regex_compile(count):
+def test_regex_compile(count, timer):
 try:
 clear_cache = re._cache.clear
 except AttributeError:
@@ -67,11 +67,11 @@
 times = []
 
 for _ in xrange(count):
- t0 = time.time()
+ t0 = timer()
 for regex, flags in regexes:
 clear_cache()
 re.compile(regex, flags)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
@@ -83,4 +83,4 @@
 util.add_standard_options_to(parser)
 options, args = parser.parse_args()
 
- util.run_benchmark(options, options.num_runs, test_regex_compile)
\ No newline at end of file
+ util.run_benchmark(options, options.num_runs, test_regex_compile)
diff --git a/performance/bm_regex_effbot.py b/performance/bm_regex_effbot.py
--- a/performance/bm_regex_effbot.py
+++ b/performance/bm_regex_effbot.py
@@ -136,7 +136,7 @@
 re.search(regexs[id], string_tables[n][id])
 
 
-def test_regex_effbot(iterations):
+def test_regex_effbot(iterations, timer):
 sizes = init_benchmarks()
 
 # Warm up.
@@ -145,10 +145,10 @@
 
 times = []
 for i in xrange(iterations):
- t0 = time.time()
+ t0 = timer()
 for size in sizes:
 run_benchmarks(size)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
diff --git a/performance/bm_regex_v8.py b/performance/bm_regex_v8.py
--- a/performance/bm_regex_v8.py
+++ b/performance/bm_regex_v8.py
@@ -1658,10 +1658,10 @@
 re.search(r'jvaqbjf', strings[63])
 
 
-def test_regex_v8(count):
+def test_regex_v8(count, timer):
 times = []
 for i in xrange(count):
- t0 = time.time()
+ t0 = timer()
 block0()
 block1()
 block2()
@@ -1674,7 +1674,7 @@
 block9()
 block10()
 block11()
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
diff --git a/performance/bm_richards.py b/performance/bm_richards.py
--- a/performance/bm_richards.py
+++ b/performance/bm_richards.py
@@ -19,16 +19,16 @@
 from compat import xrange
 
 
-def test_richards(iterations):
+def test_richards(iterations, timer):
 # Warm-up
 r = richards.Richards()
 r.run(iterations=2)
 
 times = []
 for _ in xrange(iterations):
- t0 = time.time()
+ t0 = timer()
 r.run(iterations=1)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
diff --git a/performance/bm_rietveld.py b/performance/bm_rietveld.py
--- a/performance/bm_rietveld.py
+++ b/performance/bm_rietveld.py
@@ -87,14 +87,14 @@
 return tmpl, context
 
 
-def test_rietveld(count, tmpl, context):
+def test_rietveld(count, timer, tmpl, context):
 # Warm up Django.
 tmpl.render(context)
 tmpl.render(context)
 
 times = []
 for _ in xrange(count):
- t0 = time.time()
+ t0 = timer()
 # 30 calls to render, so that we don't measure loop overhead.
 tmpl.render(context)
 tmpl.render(context)
@@ -126,7 +126,7 @@
 tmpl.render(context)
 tmpl.render(context)
 tmpl.render(context)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
diff --git a/performance/bm_spambayes.py b/performance/bm_spambayes.py
--- a/performance/bm_spambayes.py
+++ b/performance/bm_spambayes.py
@@ -21,7 +21,7 @@
 import util
 
 
-def test_spambayes(iterations, messages, ham_classifier):
+def test_spambayes(iterations, timer, messages, ham_classifier):
 # Prime the pump. This still leaves some hot functions uncompiled; these
 # will be noticed as hot during the timed loops below.
 for msg in messages:
@@ -29,10 +29,10 @@
 
 times = []
 for _ in xrange(iterations):
- t0 = time.time()
+ t0 = timer()
 for msg in messages:
 ham_classifier.score(msg)
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
diff --git a/performance/bm_spectral_norm.py b/performance/bm_spectral_norm.py
--- a/performance/bm_spectral_norm.py
+++ b/performance/bm_spectral_norm.py
@@ -40,10 +40,10 @@
 
 DEFAULT_N = 130
 
-def main(n):
+def main(n, timer):
 times = []
 for i in xrange(n):
- t0 = time.time()
+ t0 = timer()
 u = [1] * DEFAULT_N
 
 for dummy in xrange(10):
@@ -55,7 +55,7 @@
 for ue, ve in izip(u, v):
 vBv += ue * ve
 vv += ve * ve
- tk = time.time()
+ tk = timer()
 times.append(tk - t0)
 return times
 
diff --git a/performance/bm_spitfire.py b/performance/bm_spitfire.py
--- a/performance/bm_spitfire.py
+++ b/performance/bm_spitfire.py
@@ -36,7 +36,7 @@
 </table>
 """
 
-def test_spitfire(count):
+def test_spitfire(count, timer):
 # Activate the most aggressive Spitfire optimizations. While it might
 # conceivably be interesting to stress Spitfire's lower optimization
 # levels, we assume no-one will be running a production system with those
@@ -55,20 +55,20 @@
 
 times = []
 for _ in xrange(count):
- t0 = time.time()
+ t0 = timer()
 data = spitfire_tmpl_o4(search_list=[{"table": table}]).main()
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
 
-def test_spitfire_without_psyco(count):
+def test_spitfire_without_psyco(count, timer):
 class FakePsyco(object):
 def bind(self, *args, **kwargs):
 pass
 sys.modules["psyco"] = FakePsyco()
 
- return test_spitfire(count)
+ return test_spitfire(count, timer)
 
 
 if __name__ == "__main__":
diff --git a/performance/bm_telco.py b/performance/bm_telco.py
--- a/performance/bm_telco.py
+++ b/performance/bm_telco.py
@@ -20,7 +20,6 @@
 import os
 from struct import unpack
 import sys
-from time import clock as time
 
 from compat import xrange
 
@@ -32,7 +31,7 @@
 
 filename = rel_path("data", "telco-bench.b")
 
-def run():
+def run(timer):
 getcontext().rounding = ROUND_DOWN
 rates = list(map(Decimal, ('0.0013', '0.00894')))
 twodig = Decimal('0.01')
@@ -41,7 +40,7 @@
 disttax = Decimal("0.0341")
 
 infil = open(filename, "rb")
- start = time()
+ start = timer()
 
 sumT = Decimal("0") # sum of total prices
 sumB = Decimal("0") # sum of basic tax
@@ -73,14 +72,14 @@
 sumT += t
 
 infil.close()
- end = time()
+ end = timer()
 return end - start
 
-def main(n):
- run() # warmup
+def main(n, timer):
+ run(timer) # warmup
 times = []
 for i in xrange(n):
- times.append(run())
+ times.append(run(timer))
 return times
 
 
diff --git a/performance/bm_threading.py b/performance/bm_threading.py
--- a/performance/bm_threading.py
+++ b/performance/bm_threading.py
@@ -29,33 +29,33 @@
 iterations -= 1
 
 
-def test_iterative_count(iterations, num_threads):
+def test_iterative_count(iterations, timer, num_threads):
 # Warm up.
 count(1000)
 
 times = []
 for _ in xrange(iterations):
- t0 = time.time()
+ t0 = timer()
 for _ in xrange(num_threads):
 count()
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
 
-def test_threaded_count(iterations, num_threads):
+def test_threaded_count(iterations, timer, num_threads):
 # Warm up.
 count(1000)
 
 times = []
 for _ in xrange(iterations):
 threads = [threading.Thread(target=count) for _ in xrange(num_threads)]
- t0 = time.time()
+ t0 = timer()
 for thread in threads:
 thread.start()
 for thread in threads:
 thread.join()
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
diff --git a/performance/bm_tornado_http.py b/performance/bm_tornado_http.py
--- a/performance/bm_tornado_http.py
+++ b/performance/bm_tornado_http.py
@@ -57,7 +57,7 @@
 return sockets[0].getsockname()
 
 
-def test_tornado(count):
+def test_tornado(count, timer):
 loop = IOLoop.instance()
 host, port = make_http_server(loop, make_application())
 url = "http://%s:%s/" % (host, port)
@@ -67,14 +67,14 @@
 def main():
 client = AsyncHTTPClient()
 for i in xrange(count):
- t0 = time.time()
+ t0 = timer()
 futures = [client.fetch(url) for j in xrange(CONCURRENCY)]
 for fut in futures:
 resp = yield fut
 buf = resp.buffer
 buf.seek(0, 2)
 assert buf.tell() == len(CHUNK) * NCHUNKS
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 
 loop.run_sync(main)
diff --git a/performance/bm_unpack_sequence.py b/performance/bm_unpack_sequence.py
--- a/performance/bm_unpack_sequence.py
+++ b/performance/bm_unpack_sequence.py
@@ -11,10 +11,10 @@
 from compat import xrange
 
 
-def do_unpacking(iterations, to_unpack):
+def do_unpacking(iterations, timer, to_unpack):
 times = []
 for _ in xrange(iterations):
- t0 = time.time()
+ t0 = timer()
 # 400 unpackings
 a, b, c, d, e, f, g, h, i, j = to_unpack
 a, b, c, d, e, f, g, h, i, j = to_unpack
@@ -416,26 +416,26 @@
 a, b, c, d, e, f, g, h, i, j = to_unpack
 a, b, c, d, e, f, g, h, i, j = to_unpack
 a, b, c, d, e, f, g, h, i, j = to_unpack
- t1 = time.time()
+ t1 = timer()
 times.append(t1 - t0)
 return times
 
 
-def test_tuple_unpacking(iterations):
+def test_tuple_unpacking(iterations, timer):
 x = tuple(range(10))
 
- return do_unpacking(iterations, x)
+ return do_unpacking(iterations, timer, x)
 
 
-def test_list_unpacking(iterations):
+def test_list_unpacking(iterations, timer):
 x = list(range(10))
 
- return do_unpacking(iterations, x)
+ return do_unpacking(iterations, timer, x)
 
 
-def test_all(iterations):
- tuple_data = test_tuple_unpacking(iterations)
- list_data = test_list_unpacking(iterations)
+def test_all(iterations, timer):
+ tuple_data = test_tuple_unpacking(iterations, timer)
+ list_data = test_list_unpacking(iterations, timer)
 return [x + y for (x, y) in zip(tuple_data, list_data)]
 
 
diff --git a/performance/util.py b/performance/util.py
--- a/performance/util.py
+++ b/performance/util.py
@@ -6,6 +6,7 @@
 
 import math
 import operator
+import time
 
 # Local imports
 from compat import reduce, print_
@@ -21,13 +22,14 @@
 function. This should return a list of floats (benchmark execution
 times).
 """
+ timer = getattr(time, options.timer)
 if options.profile:
 import cProfile
 prof = cProfile.Profile()
- prof.runcall(bench_func, num_runs, *args)
+ prof.runcall(bench_func, num_runs, timer, *args)
 prof.print_stats(sort=options.profile_sort)
 else:
- data = bench_func(num_runs, *args)
+ data = bench_func(num_runs, timer, *args)
 if options.take_geo_mean:
 product = reduce(operator.mul, data, 1)
 print_(math.pow(product, 1.0 / len(data)))
@@ -52,3 +54,5 @@
 default="time", help="Column to sort cProfile output by.")
 parser.add_option("--take_geo_mean", action="store_true",
 help="Return the geo mean, rather than individual data.")
+ parser.add_option("--timer", action="store", default="time",
+ help="Timing function from the time module.")
-- 
Repository URL: https://hg.python.org/benchmarks

---

• Previous message: [Python-checkins] cpython (3.4): Issue #22711: improve links in new distribution docs
• Next message: [Python-checkins] cpython: Issue #22261: Add a note to PCbuild\readme.txt about MSBuild switches.
• Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

---

More information about the Python-checkins mailing list