Fri May 26 16:24:53 CEST 2006

Author: steve.holden
Date: Fri May 26 16:24:51 2006
New Revision: 46334
Added:
 sandbox/trunk/rjsh-pybench/Empty.py (contents, props changed)
Modified:
 sandbox/trunk/rjsh-pybench/Arithmetic.py
 sandbox/trunk/rjsh-pybench/Calls.py
 sandbox/trunk/rjsh-pybench/Constructs.py
 sandbox/trunk/rjsh-pybench/Dict.py
 sandbox/trunk/rjsh-pybench/Exceptions.py
 sandbox/trunk/rjsh-pybench/Imports.py
 sandbox/trunk/rjsh-pybench/Instances.py
 sandbox/trunk/rjsh-pybench/Lists.py
 sandbox/trunk/rjsh-pybench/Lookups.py
 sandbox/trunk/rjsh-pybench/Numbers.py
 sandbox/trunk/rjsh-pybench/Strings.py
 sandbox/trunk/rjsh-pybench/Tuples.py
 sandbox/trunk/rjsh-pybench/Unicode.py
 sandbox/trunk/rjsh-pybench/pybench.py
Log:
Adjust rounds count to make test take roughly euqal time.
Allow comparisons between runs with different warp factors.
Add an empty test (even though it doesn't tell us much).
Modified: sandbox/trunk/rjsh-pybench/Arithmetic.py
==============================================================================

--- sandbox/trunk/rjsh-pybench/Arithmetic.py	(original)
+++ sandbox/trunk/rjsh-pybench/Arithmetic.py	Fri May 26 16:24:51 2006
@@ -4,7 +4,7 @@
 
 version = 0.3
 operations = 5 * (3 + 5 + 5 + 3 + 3 + 3)
- rounds = 120000
+ rounds = 12000*21
 
 def test(self):
 
@@ -159,7 +159,7 @@
 
 version = 0.3
 operations = 5 * (3 + 5 + 5 + 3 + 3 + 3)
- rounds = 100000
+ rounds = 10000*30
 
 def test(self):
 
@@ -314,7 +314,7 @@
 
 version = 0.3
 operations = 5 * (3 + 5 + 5 + 3 + 3 + 3)
- rounds = 120000
+ rounds = 12000*30
 
 def test(self):
 
@@ -470,7 +470,7 @@
 
 version = 0.3
 operations = 5 * (3 + 5 + 5 + 3 + 3 + 3)
- rounds = 30000
+ rounds = 3000*32
 
 def test(self):
 
@@ -625,7 +625,7 @@
 
 version = 0.3
 operations = 5 * (3 + 5 + 5 + 3 + 3 + 3)
- rounds = 40000
+ rounds = 4000*27
 
 def test(self):
 
Modified: sandbox/trunk/rjsh-pybench/Calls.py
==============================================================================
--- sandbox/trunk/rjsh-pybench/Calls.py	(original)
+++ sandbox/trunk/rjsh-pybench/Calls.py	Fri May 26 16:24:51 2006
@@ -4,7 +4,7 @@
 
 version = 0.3
 operations = 5*(1+4+4+2)
- rounds = 60000
+ rounds = 6000*22
 
 def test(self):
 
@@ -113,7 +113,7 @@
 
 version = 0.4
 operations = 5*(2+5+5+5)
- rounds = 30000
+ rounds = 3000*24
 
 def test(self):
 
@@ -234,7 +234,7 @@
 
 version = 0.3
 operations = 5*(6 + 5 + 4)
- rounds = 20000
+ rounds = 2000*27
 
 def test(self):
 
@@ -376,7 +376,7 @@
 
 version = 0.3
 operations = 5
- rounds = 50000
+ rounds = 5000*21
 
 def test(self):
 
Modified: sandbox/trunk/rjsh-pybench/Constructs.py
==============================================================================
--- sandbox/trunk/rjsh-pybench/Constructs.py	(original)
+++ sandbox/trunk/rjsh-pybench/Constructs.py	Fri May 26 16:24:51 2006
@@ -4,7 +4,7 @@
 
 version = 0.31
 operations = 30*3 # hard to say...
- rounds = 150000
+ rounds = 15000*27
 
 def test(self):
 
@@ -471,7 +471,7 @@
 
 version = 0.3
 operations = 1000*10*5
- rounds = 150
+ rounds = 540
 
 def test(self):
 
@@ -496,7 +496,7 @@
 
 version = 0.1
 operations = 5 * 5
- rounds = 8000
+ rounds = 800*25
 
 def test(self):
 
Modified: sandbox/trunk/rjsh-pybench/Dict.py
==============================================================================
--- sandbox/trunk/rjsh-pybench/Dict.py	(original)
+++ sandbox/trunk/rjsh-pybench/Dict.py	Fri May 26 16:24:51 2006
@@ -4,7 +4,7 @@
 
 version = 0.3
 operations = 5*(5 + 5)
- rounds = 60000
+ rounds = 6000*24
 
 def test(self):
 
@@ -79,7 +79,7 @@
 
 version = 0.1
 operations = 5*(6 + 6)
- rounds = 200000
+ rounds = 20000*30
 
 def test(self):
 
@@ -257,7 +257,7 @@
 
 version = 0.1
 operations = 5*(6 + 6)
- rounds = 200000
+ rounds = 20000*19
 
 def test(self):
 
@@ -346,7 +346,7 @@
 
 version = 0.3
 operations = 5*(6 + 6 + 6 + 6)
- rounds = 50000
+ rounds = 5000*44
 
 def test(self):
 
Added: sandbox/trunk/rjsh-pybench/Empty.py
==============================================================================
--- (empty file)
+++ sandbox/trunk/rjsh-pybench/Empty.py	Fri May 26 16:24:51 2006
@@ -0,0 +1,23 @@
+from pybench import Test
+
+class EmptyTest(Test):
+ """This is just here as a potential measure of repeatability."""
+
+ version = 0.3
+ operations = 1
+ rounds = 60000
+
+ def test(self):
+
+ l = []
+ for i in xrange(self.rounds):
+ pass
+
+
+ def calibrate(self):
+
+ l = []
+
+ for i in xrange(self.rounds):
+ pass
+
Modified: sandbox/trunk/rjsh-pybench/Exceptions.py
==============================================================================
--- sandbox/trunk/rjsh-pybench/Exceptions.py	(original)
+++ sandbox/trunk/rjsh-pybench/Exceptions.py	Fri May 26 16:24:51 2006
@@ -4,7 +4,7 @@
 
 version = 0.1
 operations = 2 + 3
- rounds = 60000
+ rounds = 6000*25
 
 def test(self):
 
@@ -44,7 +44,7 @@
 
 version = 0.1
 operations = 15 * 10
- rounds = 200000
+ rounds = 20000*16
 
 def test(self):
 
Modified: sandbox/trunk/rjsh-pybench/Imports.py
==============================================================================
--- sandbox/trunk/rjsh-pybench/Imports.py	(original)
+++ sandbox/trunk/rjsh-pybench/Imports.py	Fri May 26 16:24:51 2006
@@ -8,7 +8,7 @@
 
 version = 0.1
 operations = 5 * 5
- rounds = 20000
+ rounds = 2000*15
 
 def test(self):
 
@@ -53,7 +53,7 @@
 
 version = 0.1
 operations = 5 * 5
- rounds = 20000
+ rounds = 2000*20
 
 def test(self):
 
@@ -97,7 +97,7 @@
 
 version = 0.1
 operations = 5 * 5
- rounds = 20000
+ rounds = 2000*17
 
 def test(self):
 
Modified: sandbox/trunk/rjsh-pybench/Instances.py
==============================================================================
--- sandbox/trunk/rjsh-pybench/Instances.py	(original)
+++ sandbox/trunk/rjsh-pybench/Instances.py	Fri May 26 16:24:51 2006
@@ -4,7 +4,7 @@
 
 version = 0.2
 operations = 3 + 7 + 4
- rounds = 60000
+ rounds = 6000*17
 
 def test(self):
 
Modified: sandbox/trunk/rjsh-pybench/Lists.py
==============================================================================
--- sandbox/trunk/rjsh-pybench/Lists.py	(original)
+++ sandbox/trunk/rjsh-pybench/Lists.py	Fri May 26 16:24:51 2006
@@ -4,7 +4,7 @@
 
 version = 0.3
 operations = 5* (6 + 6 + 6)
- rounds = 60000
+ rounds = 6000*45
 
 def test(self):
 
@@ -132,7 +132,7 @@
 
 version = 0.4
 operations = 25*(3+1+2+1)
- rounds = 400
+ rounds = 40*45
 
 def test(self):
 
@@ -169,7 +169,7 @@
 
 version = 0.3
 operations = 5*(1+ 6 + 6 + 3 + 1)
- rounds = 60000
+ rounds = 6000*15
 
 def test(self):
 
Modified: sandbox/trunk/rjsh-pybench/Lookups.py
==============================================================================
--- sandbox/trunk/rjsh-pybench/Lookups.py	(original)
+++ sandbox/trunk/rjsh-pybench/Lookups.py	Fri May 26 16:24:51 2006
@@ -4,7 +4,7 @@
 
 version = 0.3
 operations = 5*(12 + 12)
- rounds = 100000
+ rounds = 10000*16
 
 def test(self):
 
@@ -185,7 +185,7 @@
 
 version = 0.3
 operations = 5*(12 + 12)
- rounds = 100000
+ rounds = 10000*20
 
 def test(self):
 
@@ -371,7 +371,7 @@
 
 version = 0.3
 operations = 5*(12 + 12)
- rounds = 100000
+ rounds = 10000*14
 
 def test(self):
 
@@ -559,7 +559,7 @@
 
 version = 0.3
 operations = 5*(12 + 12)
- rounds = 100000
+ rounds = 10000*22
 
 def test(self):
 
@@ -747,7 +747,7 @@
 
 version = 0.3
 operations = 5*(3*5 + 3*5)
- rounds = 70000
+ rounds = 7000*15
 
 def test(self):
 
Modified: sandbox/trunk/rjsh-pybench/Numbers.py
==============================================================================
--- sandbox/trunk/rjsh-pybench/Numbers.py	(original)
+++ sandbox/trunk/rjsh-pybench/Numbers.py	Fri May 26 16:24:51 2006
@@ -4,7 +4,7 @@
 
 version = 0.1
 operations = 30 * 5
- rounds = 120000
+ rounds = 12000*21
 
 def test(self):
 
@@ -200,7 +200,7 @@
 
 version = 0.1
 operations = 30 * 5
- rounds = 60000
+ rounds = 6000*27
 
 def test(self):
 
@@ -396,7 +396,7 @@
 
 version = 0.1
 operations = 30 * 5
- rounds = 60000
+ rounds = 6000*16
 
 def test(self):
 
@@ -592,7 +592,7 @@
 
 version = 0.1
 operations = 30 * 5
- rounds = 60000
+ rounds = 6000*24
 
 def test(self):
 
Modified: sandbox/trunk/rjsh-pybench/Strings.py
==============================================================================
--- sandbox/trunk/rjsh-pybench/Strings.py	(original)
+++ sandbox/trunk/rjsh-pybench/Strings.py	Fri May 26 16:24:51 2006
@@ -87,7 +87,7 @@
 
 version = 0.2
 operations = 10 * 5
- rounds = 200000
+ rounds = 20000*22
 
 def test(self):
 
@@ -169,7 +169,7 @@
 
 version = 0.1
 operations = 10 * 5
- rounds = 200000
+ rounds = 20000*28
 
 def test(self):
 
@@ -251,7 +251,7 @@
 
 version = 0.1
 operations = 10 * 5
- rounds = 80000
+ rounds = 8000*32
 
 def test(self):
 
@@ -326,7 +326,7 @@
 
 version = 0.1
 operations = 5 * 7
- rounds = 100000
+ rounds = 10000*15
 
 def test(self):
 
@@ -389,7 +389,7 @@
 
 version = 0.1
 operations = 3 * (5 + 4 + 2 + 1)
- rounds = 70000
+ rounds = 140000
 
 def test(self):
 
@@ -462,7 +462,7 @@
 
 version = 0.1
 operations = 10 * 7
- rounds = 80000
+ rounds = 8000*24
 
 def test(self):
 
Modified: sandbox/trunk/rjsh-pybench/Tuples.py
==============================================================================
--- sandbox/trunk/rjsh-pybench/Tuples.py	(original)
+++ sandbox/trunk/rjsh-pybench/Tuples.py	Fri May 26 16:24:51 2006
@@ -4,7 +4,7 @@
 
 version = 0.31
 operations = 3 * 25 * 10 * 7
- rounds = 400
+ rounds = 40*15
 
 def test(self):
 
@@ -272,7 +272,7 @@
 
 version = 0.3
 operations = 5*(1 + 3 + 6 + 2)
- rounds = 80000
+ rounds = 8000*16
 
 def test(self):
 
Modified: sandbox/trunk/rjsh-pybench/Unicode.py
==============================================================================
--- sandbox/trunk/rjsh-pybench/Unicode.py	(original)
+++ sandbox/trunk/rjsh-pybench/Unicode.py	Fri May 26 16:24:51 2006
@@ -10,7 +10,7 @@
 
 version = 0.1
 operations = 10 * 5
- rounds = 60000
+ rounds = 6000*7
 
 def test(self):
 
@@ -92,7 +92,7 @@
 
 version = 0.1
 operations = 10 * 5
- rounds = 150000
+ rounds = 15000*17
 
 def test(self):
 
@@ -174,7 +174,7 @@
 
 version = 0.1
 operations = 10 * 5
- rounds = 80000
+ rounds = 8000*12
 
 def test(self):
 
@@ -310,7 +310,7 @@
 
 version = 0.1
 operations = 3 * (5 + 4 + 2 + 1)
- rounds = 10000
+ rounds = 1000*15
 
 def test(self):
 
@@ -383,7 +383,7 @@
 
 version = 0.1
 operations = 5 * 9
- rounds = 100000
+ rounds = 10000*25
 
 def test(self):
 
@@ -460,7 +460,7 @@
 
 version = 0.1
 operations = 5 * 8
- rounds = 100000
+ rounds = 10000*15
 
 def test(self):
 
Modified: sandbox/trunk/rjsh-pybench/pybench.py
==============================================================================
--- sandbox/trunk/rjsh-pybench/pybench.py	(original)
+++ sandbox/trunk/rjsh-pybench/pybench.py	Fri May 26 16:24:51 2006
@@ -35,7 +35,7 @@
 """
 
 # Version number
-__version__ = '1.3'
+__version__ = '1.4'
 
 #
 # NOTE: Use xrange for all test loops unless you want to face
@@ -115,13 +115,15 @@
 # Misc. internal variables
 last_timing = (0,0,0) # last timing (real,run,calibration)
 warp = 1 # warp factor this test uses
- cruns = 0 # number of calibration runs
+ cruns = 20 # number of calibration runs
 overhead = None # list of calibration timings
 
 def __init__(self,warp=1):
 
 if warp > 1:
 self.rounds = self.rounds / warp
+ if self.rounds == 0:
+ self.rounds = 1
 self.warp = warp
 self.times = []
 self.overhead = []
@@ -183,9 +185,13 @@
 
 def stat(self):
 
- """ Returns two value: average time per run and average per
- operation.
+ """ Returns four values:
+ minimum round time
+ average time per round
+ average time per operation
+ average overhead time
 
+ XXX Should this take warp factors into account?
 """
 runs = len(self.times)
 if runs == 0:
@@ -200,7 +206,7 @@
 else:
 # use self.last_timing - not too accurate
 ov_avg = self.last_timing[2]
- return mintime,avg,op_avg,ov_avg
+ return mintime, avg, op_avg, ov_avg
 
 ### Load Setup
 
@@ -225,7 +231,7 @@
 self.tests = {}
 self.version = 0.31
 
- def load_tests(self,setupmod,warp=1, limitnames=""):
+ def load_tests(self, setupmod, warp=1, limitnames="", verbose=0):
 
 self.warp = warp
 if limitnames:
@@ -233,7 +239,7 @@
 else:
 limitnames = None
 tests = self.tests
- print 'Searching for tests...'
+ print 'Searching for tests ...',
 setupmod.__dict__.values()
 for c in setupmod.__dict__.values():
 if not hasattr(c,'is_a_test'):
@@ -246,8 +252,12 @@
 tests[name] = c(warp)
 l = tests.keys()
 l.sort()
- for t in l:
- print ' ',t
+ if verbose:
+ print
+ for t in l:
+ print ' ',t
+ else:
+ print len(l), "found"
 print
 
 def run(self, verbose):
@@ -259,6 +269,7 @@
 print
 roundtime = clock()
 for i in range(self.rounds):
+ roundstarttime = clock()
 if verbose:
 print ' Round %-25i real abs overhead' % (i+1)
 for j in range(len(tests)):
@@ -274,7 +285,7 @@
 ((clock() - roundtime)/(i+1))
 print
 else:
- print '%d ... done'%i
+ print '%d done in %.3f seconds' % (i+1, (clock() - roundstarttime))
 self.roundtime = (clock() - roundtime) / self.rounds
 print
 
@@ -305,8 +316,8 @@
 tests.sort()
 compatible = 1
 totalmintime = other_totalmintime = 0
- for name,t in tests:
- mintime,avg,op_avg,ov_avg = t.stat()
+ for name, t in tests:
+ mintime, avg, op_avg, ov_avg = t.stat()
 totalmintime += mintime
 try:
 other = compare_to.tests[name]
@@ -314,24 +325,27 @@
 other = None
 if other and other.version == t.version and \
 other.operations == t.operations:
- mintime1,avg1,op_avg1,ov_avg1 = other.stat()
+ mintime1, avg1, op_avg1, ov_avg1 = other.stat()
 other_totalmintime += mintime1
- diff = (mintime/mintime1 - 1.0)*100.0
+ diff = ((mintime*self.warp)/(mintime1*other.warp) - 1.0)*100.0
 if hidenoise and abs(qop_avg) < 10:
 diff = ''
 else:
 diff = '%+7.2f%%' % diff
 else:
- qavg,diff = 'n/a', 'n/a'
+ qavg, diff = 'n/a', 'n/a'
 compatible = 0
 print '%30s: %8.2f ms %8.2f ms %8.2f ms %8s' % \
- (name,mintime*1000.0,mintime1*1000.0, avg*1000.0,diff)
+ (name,mintime*1000.0,mintime1*1000.0 * compare_to.warp/self.warp, avg*1000.0,diff)
 print '-'*77
+ #
+ # Summarise test results
+ #
 if compatible and compare_to.roundtime > 0 and \
 compare_to.version == self.version:
 print '%30s: %8.2f ms %8.2f ms %+7.2f%%' % \
 ('Notional minimum round time', totalmintime * 1000.0,
- other_totalmintime * 1000.0,
+ other_totalmintime * 1000.0 * compare_to.warp/self.warp,
 ((totalmintime*self.warp)/
 (other_totalmintime*compare_to.warp)-1.0)*100.0)
 else:
@@ -479,7 +493,7 @@
 # Create benchmark object
 bench = Benchmark()
 bench.rounds = rounds
- bench.load_tests(Setup,warp,limitnames)
+ bench.load_tests(Setup, warp, limitnames, verbose)
 try:
 bench.run(verbose)
 except KeyboardInterrupt:
