aboutsummaryrefslogtreecommitdiffhomepage
path: root/bench/bench_compare.py
blob: d853358d2c6a1da1487efcc4991d58c49ec37442 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
'''
Created on May 16, 2011

@author: bungeman
'''
import sys
import getopt
import bench_util

def usage():
    """Prints simple usage information."""
    
    print '-o <file> the old bench output file.'
    print '-n <file> the new bench output file.'
    print '-h causes headers to be output.'
    print '-s <stat> the type of statistical analysis used'
    print '   Not specifying is the same as -s "avg".'
    print '  avg: average of all data points'
    print '  min: minimum of all data points'
    print '  med: median of all data points'
    print '  25th: twenty-fifth percentile for all data points'
    print '-f <fieldSpec> which fields to output and in what order.'
    print '   Not specifying is the same as -f "bctondp".'
    print '  b: bench'
    print '  c: config'
    print '  t: time type'
    print '  o: old time'
    print '  n: new time'
    print '  d: diff'
    print '  p: percent diff'
    
class BenchDiff:
    """A compare between data points produced by bench.
    
    (BenchDataPoint, BenchDataPoint)"""
    def __init__(self, old, new):
        self.old = old
        self.new = new
        self.diff = old.time - new.time
        diffp = 0
        if old.time != 0:
            diffp = self.diff / old.time
        self.diffp = diffp
    
    def __repr__(self):
        return "BenchDiff(%s, %s)" % (
                   str(self.new),
                   str(self.old),
               )
        
def main():
    """Parses command line and writes output."""
    
    try:
        opts, _ = getopt.getopt(sys.argv[1:], "f:o:n:s:h")
    except getopt.GetoptError, err:
        print str(err) 
        usage()
        sys.exit(2)
    
    column_formats = {
        'b' : '{bench: >28} ',
        'c' : '{config: <4} ',
        't' : '{time_type: <4} ',
        'o' : '{old_time: >10.2f} ',
        'n' : '{new_time: >10.2f} ',
        'd' : '{diff: >+10.2f} ',
        'p' : '{diffp: >+8.1%} ',
    }
    header_formats = {
        'b' : '{bench: >28} ',
        'c' : '{config: <4} ',
        't' : '{time_type: <4} ',
        'o' : '{old_time: >10} ',
        'n' : '{new_time: >10} ',
        'd' : '{diff: >10} ',
        'p' : '{diffp: >8} ',
    }
    
    old = None
    new = None
    column_format = ""
    header_format = ""
    columns = 'bctondp'
    header = False
    stat_type = "avg"
    
    for option, value in opts:
        if option == "-o":
            old = value
        elif option == "-n":
            new = value
        elif option == "-h":
            header = True
        elif option == "-f":
            columns = value
        elif option == "-s":
            stat_type = value
        else:
            usage()
            assert False, "unhandled option"
    
    if old is None or new is None:
        usage()
        sys.exit(2)
    
    for column_char in columns:
        if column_formats[column_char]:
            column_format += column_formats[column_char]
            header_format += header_formats[column_char]
        else:
            usage()
            sys.exit(2)
    
    if header:
        print header_format.format(
            bench='bench'
            , config='conf'
            , time_type='time'
            , old_time='old'
            , new_time='new'
            , diff='diff'
            , diffp='diffP'
        )
    
    old_benches = bench_util.parse({}, open(old, 'r'), stat_type)
    new_benches = bench_util.parse({}, open(new, 'r'), stat_type)
    
    bench_diffs = []
    for old_bench in old_benches:
        #filter new_benches for benches that match old_bench
        new_bench_match = [bench for bench in new_benches
            if old_bench.bench == bench.bench and
               old_bench.config == bench.config and
               old_bench.time_type == bench.time_type
        ]
        if (len(new_bench_match) < 1):
            continue
        bench_diffs.append(BenchDiff(old_bench, new_bench_match[0]))
    
    bench_diffs.sort(key=lambda d : [d.diffp,
                                     d.old.bench,
                                     d.old.config,
                                     d.old.time_type,
                                    ])
    for bench_diff in bench_diffs:
        print column_format.format(
            bench=bench_diff.old.bench.strip()
            , config=bench_diff.old.config.strip()
            , time_type=bench_diff.old.time_type
            , old_time=bench_diff.old.time
            , new_time=bench_diff.new.time
            , diff=bench_diff.diff
            , diffp=bench_diff.diffp
        )
    
if __name__ == "__main__":
    main()