1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
|
// Protocol messages for describing the results of benchmarks and unit tests.
syntax = "proto3";
import "google/protobuf/any.proto";
option cc_enable_arenas = true;
option java_outer_classname = "TestLogProtos";
option java_multiple_files = true;
option java_package = "org.tensorflow.util.testlog";
package tensorflow;
message EntryValue {
oneof kind {
double double_value = 1;
string string_value = 2;
}
};
// Each unit test or benchmark in a test or benchmark run provides
// some set of information. Here we provide some reasonable keys
// one would expect to see, with optional key/value pairs for things
// we haven't considered.
//
// This BenchmarkEntry should be emitted by each unit test or benchmark
// reporter.
message BenchmarkEntry {
// The name of the specific benchmark or test
// (e.g. BM_AdjustContrast_gpu_B_W_H)
string name = 1;
// If a benchmark, how many iterations it was run for
int64 iters = 2;
// Total cpu time used for all iterations (in seconds)
double cpu_time = 3;
// Total wall time used for all iterations (in seconds)
double wall_time = 4;
// Throughput (in MB/s)
double throughput = 5;
// Generic map from result key to value.
map<string, EntryValue> extras = 6;
};
message BenchmarkEntries {
repeated BenchmarkEntry entry = 1;
}
message BuildConfiguration {
string mode = 1; // opt, dbg, etc
repeated string cc_flags = 2; // CC compiler flags, if known
repeated string opts = 3; // Bazel compilation options, if known
};
message CommitId {
oneof kind {
// Submitted changelist.
int64 changelist = 1;
string hash = 2;
}
// Hash of intermediate change between hash/changelist and what was tested.
// Not used if the build is from a commit without modifications.
string snapshot = 3;
// Changelist tested if the change list is not already submitted.
int64 pending_changelist = 4;
};
message CPUInfo {
int64 num_cores = 1;
int64 num_cores_allowed = 2;
// How fast are these cpus?
double mhz_per_cpu = 3;
// Additional cpu information. For example,
// Intel Ivybridge with HyperThreading (24 cores) dL1:32KB dL2:256KB dL3:30MB
string cpu_info = 4;
// What kind of cpu scaling is enabled on the host.
// Examples include "performance", "ondemand", "conservative", "mixed".
string cpu_governor = 5;
// Cache sizes (in bytes), e.g. "L2": 262144 (for 256KB)
map<string, int64> cache_size = 6;
};
message MemoryInfo {
int64 total = 1; // Total virtual memory in bytes
int64 available = 2; // Immediately available memory in bytes
}
message GPUInfo {
string model = 1; // e.g. "Tesla K40c"
string uuid = 2; // Final entry in output of "nvidia-smi -L"
string bus_id = 3; // e.g. "0000:04:00.0"
};
message PlatformInfo {
string bits = 1; // e.g. '64bit'
string linkage = 2; // e.g. 'ELF'
string machine = 3; // e.g. 'i386'
string release = 4; // e.g. '3.13.0-76-generic'
string system = 5; // e.g. 'Linux'
string version = 6; // e.g. '#120-Ubuntu SMP Mon Jan 18 15:59:10 UTC 2016'
};
message AvailableDeviceInfo { // Matches DeviceAttributes
string name = 1; // Device name.
string type = 2; // Device type, e.g. 'CPU' or 'GPU'.
int64 memory_limit = 3; // Memory capacity in bytes.
string physical_description = 4; // The physical description of this device.
};
message MachineConfiguration {
// Host name of machine that ran the benchmark.
string hostname = 1;
// Unique serial number of the machine.
string serial_identifier = 7;
// Additional platform information.
PlatformInfo platform_info = 2;
// CPU Information.
CPUInfo cpu_info = 3;
// Other devices that are attached and relevant (e.g. GPUInfo).
repeated google.protobuf.Any device_info = 4;
// Devices accessible to the test (e.g. as given by list_local_devices).
repeated AvailableDeviceInfo available_device_info = 5;
MemoryInfo memory_info = 6;
};
// Run-specific items such as arguments to the test / benchmark.
message RunConfiguration {
repeated string argument = 1;
}
// The output of one benchmark / test run. Each run contains a list of
// tests or benchmarks, stored as BenchmarkEntry messages.
//
// This message should be emitted by the reporter (which runs the
// test / BM in a subprocess and then reads the emitted BenchmarkEntry messages;
// usually from a serialized json file, finally collecting them along
// with additional information about the test run.
message TestResults {
// The target of the run, e.g.:
// //tensorflow/core:kernels_adjust_contrast_op_benchmark_test
string target = 1;
// The list of tests or benchmarks in this run.
BenchmarkEntries entries = 2;
// The configuration of the build (compiled opt? with cuda? any copts?)
BuildConfiguration build_configuration = 3;
// The commit id (git hash or changelist)
CommitId commit_id = 4;
// The time the run started (in seconds of UTC time since Unix epoch)
int64 start_time = 5;
// The amount of time the total run took (wall time in seconds)
double run_time = 6;
// Machine-specific parameters (Platform and CPU info)
MachineConfiguration machine_configuration = 7;
// Run-specific parameters (arguments, etc)
RunConfiguration run_configuration = 8;
// Benchmark target identifier.
string name = 9;
// The type of benchmark.
enum BenchmarkType {
UNKNOWN = 0; // Fallback for protos written before Type was introduced.
CPP_MICROBENCHMARK = 1;
PYTHON_BENCHMARK = 2;
ANDROID_BENCHMARK = 3;
}
BenchmarkType benchmark_type = 10;
// Used for differentiating between continuous and debug builds.
// Must be one of:
// * cbuild: results from continuous build.
// * presubmit: results from oneshot requests.
// * culprit: results from culprit finder rerun.
string run_mode = 11;
};
|
