aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools/codegen
diff options
context:
space:
mode:
authorGravatar Muxi Yan <mxyan@google.com>2017-12-13 14:04:45 -0800
committerGravatar Muxi Yan <mxyan@google.com>2017-12-13 14:04:45 -0800
commit032e9b32dc5978a042bdda5c3031ae6cbd928973 (patch)
tree9a9102f17700c05003cb2c85801913e48637ddf5 /tools/codegen
parent86ece2c8f6740bf046cddbe66e6cec12d50fb29b (diff)
parent91a851c6e1f6bc7c1dbf84ea12558d535c911252 (diff)
Merge remote-tracking branch 'upstream/master' into fix-stream-compression-config-interface
Diffstat (limited to 'tools/codegen')
-rwxr-xr-xtools/codegen/core/gen_header_frame.py154
-rw-r--r--tools/codegen/core/gen_hpack_tables.cc (renamed from tools/codegen/core/gen_hpack_tables.c)2
-rw-r--r--tools/codegen/core/gen_legal_metadata_characters.cc (renamed from tools/codegen/core/gen_legal_metadata_characters.c)2
-rw-r--r--tools/codegen/core/gen_percent_encoding_tables.cc (renamed from tools/codegen/core/gen_percent_encoding_tables.c)2
-rwxr-xr-xtools/codegen/core/gen_server_registered_method_bad_client_test_body.py74
-rwxr-xr-xtools/codegen/core/gen_settings_ids.py189
-rwxr-xr-xtools/codegen/core/gen_static_metadata.py384
-rwxr-xr-xtools/codegen/core/gen_stats_data.py705
8 files changed, 780 insertions, 732 deletions
diff --git a/tools/codegen/core/gen_header_frame.py b/tools/codegen/core/gen_header_frame.py
index 5375c14fd6..7219d4dd86 100755
--- a/tools/codegen/core/gen_header_frame.py
+++ b/tools/codegen/core/gen_header_frame.py
@@ -13,7 +13,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
"""Read from stdin a set of colon separated http headers:
:path: /foo/bar
content-type: application/grpc
@@ -24,109 +23,118 @@ import json
import sys
import argparse
+
def append_never_indexed(payload_line, n, count, key, value):
- payload_line.append(0x10)
- assert(len(key) <= 126)
- payload_line.append(len(key))
- payload_line.extend(ord(c) for c in key)
- assert(len(value) <= 126)
- payload_line.append(len(value))
- payload_line.extend(ord(c) for c in value)
+ payload_line.append(0x10)
+ assert (len(key) <= 126)
+ payload_line.append(len(key))
+ payload_line.extend(ord(c) for c in key)
+ assert (len(value) <= 126)
+ payload_line.append(len(value))
+ payload_line.extend(ord(c) for c in value)
+
def append_inc_indexed(payload_line, n, count, key, value):
- payload_line.append(0x40)
- assert(len(key) <= 126)
- payload_line.append(len(key))
- payload_line.extend(ord(c) for c in key)
- assert(len(value) <= 126)
- payload_line.append(len(value))
- payload_line.extend(ord(c) for c in value)
+ payload_line.append(0x40)
+ assert (len(key) <= 126)
+ payload_line.append(len(key))
+ payload_line.extend(ord(c) for c in key)
+ assert (len(value) <= 126)
+ payload_line.append(len(value))
+ payload_line.extend(ord(c) for c in value)
+
def append_pre_indexed(payload_line, n, count, key, value):
- payload_line.append(0x80 + 61 + count - n)
+ payload_line.append(0x80 + 61 + count - n)
+
_COMPRESSORS = {
- 'never': append_never_indexed,
- 'inc': append_inc_indexed,
- 'pre': append_pre_indexed,
+ 'never': append_never_indexed,
+ 'inc': append_inc_indexed,
+ 'pre': append_pre_indexed,
}
argp = argparse.ArgumentParser('Generate header frames')
-argp.add_argument('--set_end_stream', default=False, action='store_const', const=True)
-argp.add_argument('--no_framing', default=False, action='store_const', const=True)
-argp.add_argument('--compression', choices=sorted(_COMPRESSORS.keys()), default='never')
+argp.add_argument(
+ '--set_end_stream', default=False, action='store_const', const=True)
+argp.add_argument(
+ '--no_framing', default=False, action='store_const', const=True)
+argp.add_argument(
+ '--compression', choices=sorted(_COMPRESSORS.keys()), default='never')
argp.add_argument('--hex', default=False, action='store_const', const=True)
args = argp.parse_args()
# parse input, fill in vals
vals = []
for line in sys.stdin:
- line = line.strip()
- if line == '': continue
- if line[0] == '#': continue
- key_tail, value = line[1:].split(':')
- key = (line[0] + key_tail).strip()
- value = value.strip()
- vals.append((key, value))
+ line = line.strip()
+ if line == '': continue
+ if line[0] == '#': continue
+ key_tail, value = line[1:].split(':')
+ key = (line[0] + key_tail).strip()
+ value = value.strip()
+ vals.append((key, value))
# generate frame payload binary data
payload_bytes = []
if not args.no_framing:
- payload_bytes.append([]) # reserve space for header
+ payload_bytes.append([]) # reserve space for header
payload_len = 0
n = 0
for key, value in vals:
- payload_line = []
- _COMPRESSORS[args.compression](payload_line, n, len(vals), key, value)
- n += 1
- payload_len += len(payload_line)
- payload_bytes.append(payload_line)
+ payload_line = []
+ _COMPRESSORS[args.compression](payload_line, n, len(vals), key, value)
+ n += 1
+ payload_len += len(payload_line)
+ payload_bytes.append(payload_line)
# fill in header
if not args.no_framing:
- flags = 0x04 # END_HEADERS
- if args.set_end_stream:
- flags |= 0x01 # END_STREAM
- payload_bytes[0].extend([
- (payload_len >> 16) & 0xff,
- (payload_len >> 8) & 0xff,
- (payload_len) & 0xff,
- # header frame
- 0x01,
- # flags
- flags,
- # stream id
- 0x00,
- 0x00,
- 0x00,
- 0x01
- ])
+ flags = 0x04 # END_HEADERS
+ if args.set_end_stream:
+ flags |= 0x01 # END_STREAM
+ payload_bytes[0].extend([
+ (payload_len >> 16) & 0xff,
+ (payload_len >> 8) & 0xff,
+ (payload_len) & 0xff,
+ # header frame
+ 0x01,
+ # flags
+ flags,
+ # stream id
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x01
+ ])
hex_bytes = [ord(c) for c in "abcdefABCDEF0123456789"]
+
def esc_c(line):
- out = "\""
- last_was_hex = False
- for c in line:
- if 32 <= c < 127:
- if c in hex_bytes and last_was_hex:
- out += "\"\""
- if c != ord('"'):
- out += chr(c)
- else:
- out += "\\\""
- last_was_hex = False
- else:
- out += "\\x%02x" % c
- last_was_hex = True
- return out + "\""
+ out = "\""
+ last_was_hex = False
+ for c in line:
+ if 32 <= c < 127:
+ if c in hex_bytes and last_was_hex:
+ out += "\"\""
+ if c != ord('"'):
+ out += chr(c)
+ else:
+ out += "\\\""
+ last_was_hex = False
+ else:
+ out += "\\x%02x" % c
+ last_was_hex = True
+ return out + "\""
+
# dump bytes
if args.hex:
- all_bytes = []
- for line in payload_bytes:
- all_bytes.extend(line)
- print '{%s}' % ', '.join('0x%02x' % c for c in all_bytes)
+ all_bytes = []
+ for line in payload_bytes:
+ all_bytes.extend(line)
+ print '{%s}' % ', '.join('0x%02x' % c for c in all_bytes)
else:
- for line in payload_bytes:
- print esc_c(line)
+ for line in payload_bytes:
+ print esc_c(line)
diff --git a/tools/codegen/core/gen_hpack_tables.c b/tools/codegen/core/gen_hpack_tables.cc
index 73dfa9fbd6..0e7a7b80d8 100644
--- a/tools/codegen/core/gen_hpack_tables.c
+++ b/tools/codegen/core/gen_hpack_tables.cc
@@ -16,7 +16,7 @@
*
*/
-/* generates constant tables for hpack.c */
+/* generates constant tables for hpack.cc */
#include <assert.h>
#include <stddef.h>
diff --git a/tools/codegen/core/gen_legal_metadata_characters.c b/tools/codegen/core/gen_legal_metadata_characters.cc
index 5e292abb3b..fbabd2464f 100644
--- a/tools/codegen/core/gen_legal_metadata_characters.c
+++ b/tools/codegen/core/gen_legal_metadata_characters.cc
@@ -16,7 +16,7 @@
*
*/
-/* generates constant table for metadata.c */
+/* generates constant table for metadata.cc */
#include <stdio.h>
#include <string.h>
diff --git a/tools/codegen/core/gen_percent_encoding_tables.c b/tools/codegen/core/gen_percent_encoding_tables.cc
index 49ea5ea53b..a99024e5ed 100644
--- a/tools/codegen/core/gen_percent_encoding_tables.c
+++ b/tools/codegen/core/gen_percent_encoding_tables.cc
@@ -16,7 +16,7 @@
*
*/
-/* generates constant table for metadata.c */
+/* generates constant table for metadata.cc */
#include <stdio.h>
#include <string.h>
diff --git a/tools/codegen/core/gen_server_registered_method_bad_client_test_body.py b/tools/codegen/core/gen_server_registered_method_bad_client_test_body.py
index 8b5b618e8e..fa87c974f9 100755
--- a/tools/codegen/core/gen_server_registered_method_bad_client_test_body.py
+++ b/tools/codegen/core/gen_server_registered_method_bad_client_test_body.py
@@ -14,48 +14,42 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+
def esc_c(line):
- out = "\""
- last_was_hex = False
- for c in line:
- if 32 <= c < 127:
- if c in hex_bytes and last_was_hex:
- out += "\"\""
- if c != ord('"'):
- out += chr(c)
- else:
- out += "\\\""
- last_was_hex = False
- else:
- out += "\\x%02x" % c
- last_was_hex = True
- return out + "\""
+ out = "\""
+ last_was_hex = False
+ for c in line:
+ if 32 <= c < 127:
+ if c in hex_bytes and last_was_hex:
+ out += "\"\""
+ if c != ord('"'):
+ out += chr(c)
+ else:
+ out += "\\\""
+ last_was_hex = False
+ else:
+ out += "\\x%02x" % c
+ last_was_hex = True
+ return out + "\""
+
done = set()
for message_length in range(0, 3):
- for send_message_length in range(0, message_length + 1):
- payload = [
- 0,
- (message_length >> 24) & 0xff,
- (message_length >> 16) & 0xff,
- (message_length >> 8) & 0xff,
- (message_length) & 0xff
- ] + send_message_length * [0]
- for frame_length in range(0, len(payload) + 1):
- is_end = frame_length == len(payload) and send_message_length == message_length
- frame = [
- (frame_length >> 16) & 0xff,
- (frame_length >> 8) & 0xff,
- (frame_length) & 0xff,
- 0,
- 1 if is_end else 0,
- 0, 0, 0, 1
- ] + payload[0:frame_length]
- text = esc_c(frame)
- if text not in done:
- print 'GRPC_RUN_BAD_CLIENT_TEST(verifier_%s, PFX_STR %s, %s);' % (
- 'succeeds' if is_end else 'fails',
- text,
- '0' if is_end else 'GRPC_BAD_CLIENT_DISCONNECT')
- done.add(text)
+ for send_message_length in range(0, message_length + 1):
+ payload = [
+ 0, (message_length >> 24) & 0xff, (message_length >> 16) & 0xff,
+ (message_length >> 8) & 0xff, (message_length) & 0xff
+ ] + send_message_length * [0]
+ for frame_length in range(0, len(payload) + 1):
+ is_end = frame_length == len(
+ payload) and send_message_length == message_length
+ frame = [(frame_length >> 16) & 0xff, (frame_length >> 8) & 0xff,
+ (frame_length) & 0xff, 0, 1
+ if is_end else 0, 0, 0, 0, 1] + payload[0:frame_length]
+ text = esc_c(frame)
+ if text not in done:
+ print 'GRPC_RUN_BAD_CLIENT_TEST(verifier_%s, PFX_STR %s, %s);' % (
+ 'succeeds' if is_end else 'fails', text, '0'
+ if is_end else 'GRPC_BAD_CLIENT_DISCONNECT')
+ done.add(text)
diff --git a/tools/codegen/core/gen_settings_ids.py b/tools/codegen/core/gen_settings_ids.py
index 481c4216a3..bc4380605a 100755
--- a/tools/codegen/core/gen_settings_ids.py
+++ b/tools/codegen/core/gen_settings_ids.py
@@ -24,92 +24,114 @@ Setting = collections.namedtuple('Setting', 'id default min max on_error')
OnError = collections.namedtuple('OnError', 'behavior code')
clamp_invalid_value = OnError('CLAMP_INVALID_VALUE', 'PROTOCOL_ERROR')
disconnect_on_invalid_value = lambda e: OnError('DISCONNECT_ON_INVALID_VALUE', e)
-DecoratedSetting = collections.namedtuple('DecoratedSetting', 'enum name setting')
+DecoratedSetting = collections.namedtuple('DecoratedSetting',
+ 'enum name setting')
_SETTINGS = {
- 'HEADER_TABLE_SIZE': Setting(1, 4096, 0, 0xffffffff, clamp_invalid_value),
- 'ENABLE_PUSH': Setting(2, 1, 0, 1, disconnect_on_invalid_value('PROTOCOL_ERROR')),
- 'MAX_CONCURRENT_STREAMS': Setting(3, 0xffffffff, 0, 0xffffffff, disconnect_on_invalid_value('PROTOCOL_ERROR')),
- 'INITIAL_WINDOW_SIZE': Setting(4, 65535, 0, 0x7fffffff, disconnect_on_invalid_value('FLOW_CONTROL_ERROR')),
- 'MAX_FRAME_SIZE': Setting(5, 16384, 16384, 16777215, disconnect_on_invalid_value('PROTOCOL_ERROR')),
- 'MAX_HEADER_LIST_SIZE': Setting(6, _MAX_HEADER_LIST_SIZE, 0, _MAX_HEADER_LIST_SIZE, clamp_invalid_value),
- 'GRPC_ALLOW_TRUE_BINARY_METADATA': Setting(0xfe03, 0, 0, 1, clamp_invalid_value),
+ 'HEADER_TABLE_SIZE':
+ Setting(1, 4096, 0, 0xffffffff, clamp_invalid_value),
+ 'ENABLE_PUSH':
+ Setting(2, 1, 0, 1, disconnect_on_invalid_value('PROTOCOL_ERROR')),
+ 'MAX_CONCURRENT_STREAMS':
+ Setting(3, 0xffffffff, 0, 0xffffffff,
+ disconnect_on_invalid_value('PROTOCOL_ERROR')),
+ 'INITIAL_WINDOW_SIZE':
+ Setting(4, 65535, 0, 0x7fffffff,
+ disconnect_on_invalid_value('FLOW_CONTROL_ERROR')),
+ 'MAX_FRAME_SIZE':
+ Setting(5, 16384, 16384, 16777215,
+ disconnect_on_invalid_value('PROTOCOL_ERROR')),
+ 'MAX_HEADER_LIST_SIZE':
+ Setting(6, _MAX_HEADER_LIST_SIZE, 0, _MAX_HEADER_LIST_SIZE,
+ clamp_invalid_value),
+ 'GRPC_ALLOW_TRUE_BINARY_METADATA':
+ Setting(0xfe03, 0, 0, 1, clamp_invalid_value),
}
H = open('src/core/ext/transport/chttp2/transport/http2_settings.h', 'w')
C = open('src/core/ext/transport/chttp2/transport/http2_settings.c', 'w')
+
# utility: print a big comment block into a set of files
def put_banner(files, banner):
- for f in files:
- print >>f, '/*'
- for line in banner:
- print >>f, ' * %s' % line
- print >>f, ' */'
- print >>f
+ for f in files:
+ print >> f, '/*'
+ for line in banner:
+ print >> f, ' * %s' % line
+ print >> f, ' */'
+ print >> f
+
# copy-paste copyright notice from this file
with open(sys.argv[0]) as my_source:
- copyright = []
- for line in my_source:
- if line[0] != '#': break
- for line in my_source:
- if line[0] == '#':
- copyright.append(line)
- break
- for line in my_source:
- if line[0] != '#':
- break
- copyright.append(line)
- put_banner([H,C], [line[2:].rstrip() for line in copyright])
-
-put_banner([H,C], ["Automatically generated by tools/codegen/core/gen_settings_ids.py"])
-
-print >>H, "#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_SETTINGS_H"
-print >>H, "#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_SETTINGS_H"
-print >>H
-print >>H, "#include <stdint.h>"
-print >>H, "#include <stdbool.h>"
-print >>H
-
-print >>C, "#include \"src/core/ext/transport/chttp2/transport/http2_settings.h\""
-print >>C
-print >>C, "#include <grpc/support/useful.h>"
-print >>C, "#include \"src/core/lib/transport/http2_errors.h\""
-print >>C
+ copyright = []
+ for line in my_source:
+ if line[0] != '#': break
+ for line in my_source:
+ if line[0] == '#':
+ copyright.append(line)
+ break
+ for line in my_source:
+ if line[0] != '#':
+ break
+ copyright.append(line)
+ put_banner([H, C], [line[2:].rstrip() for line in copyright])
+
+put_banner(
+ [H, C],
+ ["Automatically generated by tools/codegen/core/gen_settings_ids.py"])
+
+print >> H, "#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_SETTINGS_H"
+print >> H, "#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_SETTINGS_H"
+print >> H
+print >> H, "#include <stdint.h>"
+print >> H, "#include <stdbool.h>"
+print >> H
+
+print >> C, "#include \"src/core/ext/transport/chttp2/transport/http2_settings.h\""
+print >> C
+print >> C, "#include <grpc/support/useful.h>"
+print >> C, "#include \"src/core/lib/transport/http2_errors.h\""
+print >> C
p = perfection.hash_parameters(sorted(x.id for x in _SETTINGS.values()))
print p
+
def hash(i):
- i += p.offset
- x = i % p.t
- y = i / p.t
- return x + p.r[y]
+ i += p.offset
+ x = i % p.t
+ y = i / p.t
+ return x + p.r[y]
+
-decorated_settings = [DecoratedSetting(hash(setting.id), name, setting)
- for name, setting in _SETTINGS.iteritems()]
+decorated_settings = [
+ DecoratedSetting(hash(setting.id), name, setting)
+ for name, setting in _SETTINGS.iteritems()
+]
-print >>H, 'typedef enum {'
+print >> H, 'typedef enum {'
for decorated_setting in sorted(decorated_settings):
- print >>H, ' GRPC_CHTTP2_SETTINGS_%s = %d, /* wire id %d */' % (
- decorated_setting.name, decorated_setting.enum, decorated_setting.setting.id)
-print >>H, '} grpc_chttp2_setting_id;'
-print >>H
-print >>H, '#define GRPC_CHTTP2_NUM_SETTINGS %d' % (max(x.enum for x in decorated_settings) + 1)
-
-print >>H, 'extern const uint16_t grpc_setting_id_to_wire_id[];'
-print >>C, 'const uint16_t grpc_setting_id_to_wire_id[] = {%s};' % ','.join(
+ print >> H, ' GRPC_CHTTP2_SETTINGS_%s = %d, /* wire id %d */' % (
+ decorated_setting.name, decorated_setting.enum,
+ decorated_setting.setting.id)
+print >> H, '} grpc_chttp2_setting_id;'
+print >> H
+print >> H, '#define GRPC_CHTTP2_NUM_SETTINGS %d' % (
+ max(x.enum for x in decorated_settings) + 1)
+
+print >> H, 'extern const uint16_t grpc_setting_id_to_wire_id[];'
+print >> C, 'const uint16_t grpc_setting_id_to_wire_id[] = {%s};' % ','.join(
'%d' % s for s in p.slots)
-print >>H
-print >>H, "bool grpc_wire_id_to_setting_id(uint32_t wire_id, grpc_chttp2_setting_id *out);"
+print >> H
+print >> H, "bool grpc_wire_id_to_setting_id(uint32_t wire_id, grpc_chttp2_setting_id *out);"
cgargs = {
- 'r': ','.join('%d' % (r if r is not None else 0) for r in p.r),
- 't': p.t,
- 'offset': abs(p.offset),
- 'offset_sign': '+' if p.offset > 0 else '-'
- }
-print >>C, """
+ 'r': ','.join('%d' % (r if r is not None else 0) for r in p.r),
+ 't': p.t,
+ 'offset': abs(p.offset),
+ 'offset_sign': '+' if p.offset > 0 else '-'
+}
+print >> C, """
bool grpc_wire_id_to_setting_id(uint32_t wire_id, grpc_chttp2_setting_id *out) {
uint32_t i = wire_id %(offset_sign)s %(offset)d;
uint32_t x = i %% %(t)d;
@@ -118,17 +140,17 @@ bool grpc_wire_id_to_setting_id(uint32_t wire_id, grpc_chttp2_setting_id *out) {
switch (y) {
""" % cgargs
for i, r in enumerate(p.r):
- if not r: continue
- if r < 0: print >>C, 'case %d: h -= %d; break;' % (i, -r)
- else: print >>C, 'case %d: h += %d; break;' % (i, r)
-print >>C, """
+ if not r: continue
+ if r < 0: print >> C, 'case %d: h -= %d; break;' % (i, -r)
+ else: print >> C, 'case %d: h += %d; break;' % (i, r)
+print >> C, """
}
*out = (grpc_chttp2_setting_id)h;
return h < GPR_ARRAY_SIZE(grpc_setting_id_to_wire_id) && grpc_setting_id_to_wire_id[h] == wire_id;
}
""" % cgargs
-print >>H, """
+print >> H, """
typedef enum {
GRPC_CHTTP2_CLAMP_INVALID_VALUE,
GRPC_CHTTP2_DISCONNECT_ON_INVALID_VALUE
@@ -145,25 +167,22 @@ typedef struct {
extern const grpc_chttp2_setting_parameters grpc_chttp2_settings_parameters[GRPC_CHTTP2_NUM_SETTINGS];
"""
-print >>C, "const grpc_chttp2_setting_parameters grpc_chttp2_settings_parameters[GRPC_CHTTP2_NUM_SETTINGS] = {"
+print >> C, "const grpc_chttp2_setting_parameters grpc_chttp2_settings_parameters[GRPC_CHTTP2_NUM_SETTINGS] = {"
i = 0
for decorated_setting in sorted(decorated_settings):
- while i < decorated_setting.enum:
- print >>C, "{NULL, 0, 0, 0, GRPC_CHTTP2_DISCONNECT_ON_INVALID_VALUE, GRPC_HTTP2_PROTOCOL_ERROR},"
+ while i < decorated_setting.enum:
+ print >> C, "{NULL, 0, 0, 0, GRPC_CHTTP2_DISCONNECT_ON_INVALID_VALUE, GRPC_HTTP2_PROTOCOL_ERROR},"
+ i += 1
+ print >> C, "{\"%s\", %du, %du, %du, GRPC_CHTTP2_%s, GRPC_HTTP2_%s}," % (
+ decorated_setting.name, decorated_setting.setting.default,
+ decorated_setting.setting.min, decorated_setting.setting.max,
+ decorated_setting.setting.on_error.behavior,
+ decorated_setting.setting.on_error.code,)
i += 1
- print >>C, "{\"%s\", %du, %du, %du, GRPC_CHTTP2_%s, GRPC_HTTP2_%s}," % (
- decorated_setting.name,
- decorated_setting.setting.default,
- decorated_setting.setting.min,
- decorated_setting.setting.max,
- decorated_setting.setting.on_error.behavior,
- decorated_setting.setting.on_error.code,
- )
- i += 1
-print >>C, "};"
-
-print >>H
-print >>H, "#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_SETTINGS_H */"
+print >> C, "};"
+
+print >> H
+print >> H, "#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_SETTINGS_H */"
H.close()
C.close()
diff --git a/tools/codegen/core/gen_static_metadata.py b/tools/codegen/core/gen_static_metadata.py
index 0e8cb48d0c..10903a1134 100755
--- a/tools/codegen/core/gen_static_metadata.py
+++ b/tools/codegen/core/gen_static_metadata.py
@@ -176,65 +176,66 @@ STREAM_COMPRESSION_ALGORITHMS = [
'gzip',
]
+
# utility: mangle the name of a config
def mangle(elem, name=None):
- xl = {
- '-': '_',
- ':': '',
- '/': 'slash',
- '.': 'dot',
- ',': 'comma',
- ' ': '_',
- }
+ xl = {
+ '-': '_',
+ ':': '',
+ '/': 'slash',
+ '.': 'dot',
+ ',': 'comma',
+ ' ': '_',
+ }
- def m0(x):
- if not x:
- return 'empty'
- r = ''
- for c in x:
- put = xl.get(c, c.lower())
- if not put:
- continue
- last_is_underscore = r[-1] == '_' if r else True
- if last_is_underscore and put == '_':
- continue
- elif len(put) > 1:
- if not last_is_underscore:
- r += '_'
- r += put
- r += '_'
- else:
- r += put
- if r[-1] == '_':
- r = r[:-1]
- return r
-
- def n(default, name=name):
- if name is None:
- return 'grpc_%s_' % default
- if name == '':
- return ''
- return 'grpc_%s_' % name
-
- if isinstance(elem, tuple):
- return '%s%s_%s' % (n('mdelem'), m0(elem[0]), m0(elem[1]))
- else:
- return '%s%s' % (n('mdstr'), m0(elem))
+ def m0(x):
+ if not x:
+ return 'empty'
+ r = ''
+ for c in x:
+ put = xl.get(c, c.lower())
+ if not put:
+ continue
+ last_is_underscore = r[-1] == '_' if r else True
+ if last_is_underscore and put == '_':
+ continue
+ elif len(put) > 1:
+ if not last_is_underscore:
+ r += '_'
+ r += put
+ r += '_'
+ else:
+ r += put
+ if r[-1] == '_':
+ r = r[:-1]
+ return r
+
+ def n(default, name=name):
+ if name is None:
+ return 'grpc_%s_' % default
+ if name == '':
+ return ''
+ return 'grpc_%s_' % name
+
+ if isinstance(elem, tuple):
+ return '%s%s_%s' % (n('mdelem'), m0(elem[0]), m0(elem[1]))
+ else:
+ return '%s%s' % (n('mdstr'), m0(elem))
# utility: generate some hash value for a string
def fake_hash(elem):
- return hashlib.md5(elem).hexdigest()[0:8]
+ return hashlib.md5(elem).hexdigest()[0:8]
# utility: print a big comment block into a set of files
def put_banner(files, banner):
- for f in files:
- print >> f, '/*'
- for line in banner:
- print >> f, ' * %s' % line
- print >> f, ' */'
- print >> f
+ for f in files:
+ print >> f, '/*'
+ for line in banner:
+ print >> f, ' * %s' % line
+ print >> f, ' */'
+ print >> f
# build a list of all the strings we need
@@ -244,43 +245,43 @@ static_userdata = {}
# put metadata batch callouts first, to make the check of if a static metadata
# string is a callout trivial
for elem, _ in METADATA_BATCH_CALLOUTS:
- if elem not in all_strs:
- all_strs.append(elem)
-for elem in CONFIG:
- if isinstance(elem, tuple):
- if elem[0] not in all_strs:
- all_strs.append(elem[0])
- if elem[1] not in all_strs:
- all_strs.append(elem[1])
- if elem not in all_elems:
- all_elems.append(elem)
- else:
if elem not in all_strs:
- all_strs.append(elem)
+ all_strs.append(elem)
+for elem in CONFIG:
+ if isinstance(elem, tuple):
+ if elem[0] not in all_strs:
+ all_strs.append(elem[0])
+ if elem[1] not in all_strs:
+ all_strs.append(elem[1])
+ if elem not in all_elems:
+ all_elems.append(elem)
+ else:
+ if elem not in all_strs:
+ all_strs.append(elem)
compression_elems = []
for mask in range(1, 1 << len(COMPRESSION_ALGORITHMS)):
- val = ','.join(COMPRESSION_ALGORITHMS[alg]
- for alg in range(0, len(COMPRESSION_ALGORITHMS))
- if (1 << alg) & mask)
- elem = ('grpc-accept-encoding', val)
- if val not in all_strs:
- all_strs.append(val)
- if elem not in all_elems:
- all_elems.append(elem)
- compression_elems.append(elem)
- static_userdata[elem] = 1 + (mask | 1)
+ val = ','.join(COMPRESSION_ALGORITHMS[alg]
+ for alg in range(0, len(COMPRESSION_ALGORITHMS))
+ if (1 << alg) & mask)
+ elem = ('grpc-accept-encoding', val)
+ if val not in all_strs:
+ all_strs.append(val)
+ if elem not in all_elems:
+ all_elems.append(elem)
+ compression_elems.append(elem)
+ static_userdata[elem] = 1 + (mask | 1)
stream_compression_elems = []
for mask in range(1, 1 << len(STREAM_COMPRESSION_ALGORITHMS)):
- val = ','.join(STREAM_COMPRESSION_ALGORITHMS[alg]
- for alg in range(0, len(STREAM_COMPRESSION_ALGORITHMS))
- if (1 << alg) & mask)
- elem = ('accept-encoding', val)
- if val not in all_strs:
- all_strs.append(val)
- if elem not in all_elems:
- all_elems.append(elem)
- stream_compression_elems.append(elem)
- static_userdata[elem] = 1 + (mask | 1)
+ val = ','.join(STREAM_COMPRESSION_ALGORITHMS[alg]
+ for alg in range(0, len(STREAM_COMPRESSION_ALGORITHMS))
+ if (1 << alg) & mask)
+ elem = ('accept-encoding', val)
+ if val not in all_strs:
+ all_strs.append(val)
+ if elem not in all_elems:
+ all_elems.append(elem)
+ stream_compression_elems.append(elem)
+ static_userdata[elem] = 1 + (mask | 1)
# output configuration
args = sys.argv[1:]
@@ -288,62 +289,62 @@ H = None
C = None
D = None
if args:
- if 'header' in args:
- H = sys.stdout
- else:
- H = open('/dev/null', 'w')
- if 'source' in args:
- C = sys.stdout
- else:
- C = open('/dev/null', 'w')
- if 'dictionary' in args:
- D = sys.stdout
- else:
- D = open('/dev/null', 'w')
+ if 'header' in args:
+ H = sys.stdout
+ else:
+ H = open('/dev/null', 'w')
+ if 'source' in args:
+ C = sys.stdout
+ else:
+ C = open('/dev/null', 'w')
+ if 'dictionary' in args:
+ D = sys.stdout
+ else:
+ D = open('/dev/null', 'w')
else:
- H = open(
- os.path.join(
- os.path.dirname(sys.argv[0]),
- '../../../src/core/lib/transport/static_metadata.h'), 'w')
- C = open(
- os.path.join(
- os.path.dirname(sys.argv[0]),
- '../../../src/core/lib/transport/static_metadata.cc'), 'w')
- D = open(
- os.path.join(
- os.path.dirname(sys.argv[0]),
- '../../../test/core/end2end/fuzzers/hpack.dictionary'), 'w')
+ H = open(
+ os.path.join(
+ os.path.dirname(sys.argv[0]),
+ '../../../src/core/lib/transport/static_metadata.h'), 'w')
+ C = open(
+ os.path.join(
+ os.path.dirname(sys.argv[0]),
+ '../../../src/core/lib/transport/static_metadata.cc'), 'w')
+ D = open(
+ os.path.join(
+ os.path.dirname(sys.argv[0]),
+ '../../../test/core/end2end/fuzzers/hpack.dictionary'), 'w')
# copy-paste copyright notice from this file
with open(sys.argv[0]) as my_source:
- copyright = []
- for line in my_source:
- if line[0] != '#':
- break
- for line in my_source:
- if line[0] == '#':
- copyright.append(line)
- break
- for line in my_source:
- if line[0] != '#':
- break
- copyright.append(line)
- put_banner([H, C], [line[2:].rstrip() for line in copyright])
+ copyright = []
+ for line in my_source:
+ if line[0] != '#':
+ break
+ for line in my_source:
+ if line[0] == '#':
+ copyright.append(line)
+ break
+ for line in my_source:
+ if line[0] != '#':
+ break
+ copyright.append(line)
+ put_banner([H, C], [line[2:].rstrip() for line in copyright])
hex_bytes = [ord(c) for c in 'abcdefABCDEF0123456789']
def esc_dict(line):
- out = "\""
- for c in line:
- if 32 <= c < 127:
- if c != ord('"'):
- out += chr(c)
- else:
- out += "\\\""
- else:
- out += '\\x%02X' % c
- return out + "\""
+ out = "\""
+ for c in line:
+ if 32 <= c < 127:
+ if c != ord('"'):
+ out += chr(c)
+ else:
+ out += "\\\""
+ else:
+ out += '\\x%02X' % c
+ return out + "\""
put_banner([H, C], """WARNING: Auto-generated code.
@@ -351,17 +352,13 @@ put_banner([H, C], """WARNING: Auto-generated code.
To make changes to this file, change
tools/codegen/core/gen_static_metadata.py, and then re-run it.
-See metadata.h for an explanation of the interface here, and metadata.c for
+See metadata.h for an explanation of the interface here, and metadata.cc for
an explanation of what's going on.
""".splitlines())
print >> H, '#ifndef GRPC_CORE_LIB_TRANSPORT_STATIC_METADATA_H'
print >> H, '#define GRPC_CORE_LIB_TRANSPORT_STATIC_METADATA_H'
print >> H
-print >> H, '#ifdef __cplusplus'
-print >> H, 'extern "C" {'
-print >> H, '#endif'
-print >> H
print >> H, '#include "src/core/lib/transport/metadata.h"'
print >> H
@@ -373,27 +370,26 @@ print >> C
str_ofs = 0
id2strofs = {}
for i, elem in enumerate(all_strs):
- id2strofs[i] = str_ofs
- str_ofs += len(elem)
+ id2strofs[i] = str_ofs
+ str_ofs += len(elem)
def slice_def(i):
- return ('{&grpc_static_metadata_refcounts[%d],'
- ' {{g_bytes+%d, %d}}}') % (
- i, id2strofs[i], len(all_strs[i]))
+ return ('{&grpc_static_metadata_refcounts[%d],'
+ ' {{g_bytes+%d, %d}}}') % (i, id2strofs[i], len(all_strs[i]))
# validate configuration
for elem, _ in METADATA_BATCH_CALLOUTS:
- assert elem in all_strs
+ assert elem in all_strs
print >> H, '#define GRPC_STATIC_MDSTR_COUNT %d' % len(all_strs)
print >> H, ('extern const grpc_slice '
'grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT];')
for i, elem in enumerate(all_strs):
- print >> H, '/* "%s" */' % elem
- print >> H, '#define %s (grpc_static_slice_table[%d])' % (
- mangle(elem).upper(), i)
+ print >> H, '/* "%s" */' % elem
+ print >> H, '#define %s (grpc_static_slice_table[%d])' % (
+ mangle(elem).upper(), i)
print >> H
print >> C, 'static uint8_t g_bytes[] = {%s};' % (
','.join('%d' % ord(c) for c in ''.join(all_strs)))
@@ -415,7 +411,7 @@ print >> H, ('extern grpc_slice_refcount '
print >> C, ('grpc_slice_refcount '
'grpc_static_metadata_refcounts[GRPC_STATIC_MDSTR_COUNT] = {')
for i, elem in enumerate(all_strs):
- print >> C, ' {&grpc_static_metadata_vtable, &static_sub_refcnt},'
+ print >> C, ' {&grpc_static_metadata_vtable, &static_sub_refcnt},'
print >> C, '};'
print >> C
print >> H, '#define GRPC_IS_STATIC_METADATA_STRING(slice) \\'
@@ -425,7 +421,7 @@ print >> H
print >> C, ('const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT]'
' = {')
for i, elem in enumerate(all_strs):
- print >> C, slice_def(i) + ','
+ print >> C, slice_def(i) + ','
print >> C, '};'
print >> C
print >> H, '#define GRPC_STATIC_METADATA_INDEX(static_slice) \\'
@@ -435,10 +431,10 @@ print >> H
print >> D, '# hpack fuzzing dictionary'
for i, elem in enumerate(all_strs):
- print >> D, '%s' % (esc_dict([len(elem)] + [ord(c) for c in elem]))
+ print >> D, '%s' % (esc_dict([len(elem)] + [ord(c) for c in elem]))
for i, elem in enumerate(all_elems):
- print >> D, '%s' % (esc_dict([0, len(elem[0])] + [ord(c) for c in elem[0]] +
- [len(elem[1])] + [ord(c) for c in elem[1]]))
+ print >> D, '%s' % (esc_dict([0, len(elem[0])] + [ord(c) for c in elem[0]] +
+ [len(elem[1])] + [ord(c) for c in elem[1]]))
print >> H, '#define GRPC_STATIC_MDELEM_COUNT %d' % len(all_elems)
print >> H, ('extern grpc_mdelem_data '
@@ -446,10 +442,9 @@ print >> H, ('extern grpc_mdelem_data '
print >> H, ('extern uintptr_t '
'grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT];')
for i, elem in enumerate(all_elems):
- print >> H, '/* "%s": "%s" */' % elem
- print >> H, ('#define %s (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[%d], '
- 'GRPC_MDELEM_STORAGE_STATIC))') % (
- mangle(elem).upper(), i)
+ print >> H, '/* "%s": "%s" */' % elem
+ print >> H, ('#define %s (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[%d], '
+ 'GRPC_MDELEM_STORAGE_STATIC))') % (mangle(elem).upper(), i)
print >> H
print >> C, ('uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] '
'= {')
@@ -460,42 +455,38 @@ print >> C
def str_idx(s):
- for i, s2 in enumerate(all_strs):
- if s == s2:
- return i
+ for i, s2 in enumerate(all_strs):
+ if s == s2:
+ return i
def md_idx(m):
- for i, m2 in enumerate(all_elems):
- if m == m2:
- return i
+ for i, m2 in enumerate(all_elems):
+ if m == m2:
+ return i
def offset_trials(mink):
- yield 0
- for i in range(1, 100):
- for mul in [-1, 1]:
- yield mul * i
+ yield 0
+ for i in range(1, 100):
+ for mul in [-1, 1]:
+ yield mul * i
def perfect_hash(keys, name):
- p = perfection.hash_parameters(keys)
-
- def f(i, p=p):
- i += p.offset
- x = i % p.t
- y = i / p.t
- return x + p.r[y]
-
- return {
- 'PHASHRANGE':
- p.t - 1 + max(p.r),
- 'PHASHNKEYS':
- len(p.slots),
- 'pyfunc':
- f,
- 'code':
- """
+ p = perfection.hash_parameters(keys)
+
+ def f(i, p=p):
+ i += p.offset
+ x = i % p.t
+ y = i / p.t
+ return x + p.r[y]
+
+ return {
+ 'PHASHRANGE': p.t - 1 + max(p.r),
+ 'PHASHNKEYS': len(p.slots),
+ 'pyfunc': f,
+ 'code': """
static const int8_t %(name)s_r[] = {%(r)s};
static uint32_t %(name)s_phash(uint32_t i) {
i %(offset_sign)s= %(offset)d;
@@ -509,13 +500,13 @@ static uint32_t %(name)s_phash(uint32_t i) {
return h;
}
""" % {
- 'name': name,
- 'r': ','.join('%d' % (r if r is not None else 0) for r in p.r),
- 't': p.t,
- 'offset': abs(p.offset),
- 'offset_sign': '+' if p.offset > 0 else '-'
+ 'name': name,
+ 'r': ','.join('%d' % (r if r is not None else 0) for r in p.r),
+ 't': p.t,
+ 'offset': abs(p.offset),
+ 'offset_sign': '+' if p.offset > 0 else '-'
+ }
}
- }
elem_keys = [
@@ -527,14 +518,14 @@ print >> C, elem_hash['code']
keys = [0] * int(elem_hash['PHASHRANGE'])
idxs = [255] * int(elem_hash['PHASHNKEYS'])
for i, k in enumerate(elem_keys):
- h = elem_hash['pyfunc'](k)
- assert keys[h] == 0
- keys[h] = k
- idxs[h] = i
+ h = elem_hash['pyfunc'](k)
+ assert keys[h] == 0
+ keys[h] = k
+ idxs[h] = i
print >> C, 'static const uint16_t elem_keys[] = {%s};' % ','.join(
'%d' % k for k in keys)
-print >> C, 'static const uint8_t elem_idxs[] = {%s};' % ','.join(
- '%d' % i for i in idxs)
+print >> C, 'static const uint8_t elem_idxs[] = {%s};' % ','.join('%d' % i
+ for i in idxs)
print >> C
print >> H, 'grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b);'
@@ -548,12 +539,12 @@ print >> C
print >> C, 'grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT] = {'
for a, b in all_elems:
- print >> C, '{%s,%s},' % (slice_def(str_idx(a)), slice_def(str_idx(b)))
+ print >> C, '{%s,%s},' % (slice_def(str_idx(a)), slice_def(str_idx(b)))
print >> C, '};'
print >> H, 'typedef enum {'
for elem, _ in METADATA_BATCH_CALLOUTS:
- print >> H, ' %s,' % mangle(elem, 'batch').upper()
+ print >> H, ' %s,' % mangle(elem, 'batch').upper()
print >> H, ' GRPC_BATCH_CALLOUTS_COUNT'
print >> H, '} grpc_metadata_batch_callouts_index;'
print >> H
@@ -561,7 +552,7 @@ print >> H, 'typedef union {'
print >> H, ' struct grpc_linked_mdelem *array[GRPC_BATCH_CALLOUTS_COUNT];'
print >> H, ' struct {'
for elem, _ in METADATA_BATCH_CALLOUTS:
- print >> H, ' struct grpc_linked_mdelem *%s;' % mangle(elem, '').lower()
+ print >> H, ' struct grpc_linked_mdelem *%s;' % mangle(elem, '').lower()
print >> H, ' } named;'
print >> H, '} grpc_metadata_batch_callouts;'
print >> H
@@ -573,7 +564,7 @@ print >> H, ('extern bool grpc_static_callout_is_default['
print >> H
print >> C, 'bool grpc_static_callout_is_default[GRPC_BATCH_CALLOUTS_COUNT] = {'
for elem, is_default in METADATA_BATCH_CALLOUTS:
- print >> C, ' %s, // %s' % (str(is_default).lower(), elem)
+ print >> C, ' %s, // %s' % (str(is_default).lower(), elem)
print >> C, '};'
print >> C
@@ -592,15 +583,12 @@ print >> H, 'extern const uint8_t grpc_static_accept_stream_encoding_metadata[%d
1 << len(STREAM_COMPRESSION_ALGORITHMS))
print >> C, 'const uint8_t grpc_static_accept_stream_encoding_metadata[%d] = {' % (
1 << len(STREAM_COMPRESSION_ALGORITHMS))
-print >> C, '0,%s' % ','.join('%d' % md_idx(elem) for elem in stream_compression_elems)
+print >> C, '0,%s' % ','.join('%d' % md_idx(elem)
+ for elem in stream_compression_elems)
print >> C, '};'
print >> H, '#define GRPC_MDELEM_ACCEPT_STREAM_ENCODING_FOR_ALGORITHMS(algs) (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[grpc_static_accept_stream_encoding_metadata[(algs)]], GRPC_MDELEM_STORAGE_STATIC))'
-print >> H, '#ifdef __cplusplus'
-print >> H, '}'
-print >> H, '#endif'
-
print >> H, '#endif /* GRPC_CORE_LIB_TRANSPORT_STATIC_METADATA_H */'
H.close()
diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py
index d439d99a8c..93f2035c45 100755
--- a/tools/codegen/core/gen_stats_data.py
+++ b/tools/codegen/core/gen_stats_data.py
@@ -22,397 +22,436 @@ import yaml
import json
with open('src/core/lib/debug/stats_data.yaml') as f:
- attrs = yaml.load(f.read())
+ attrs = yaml.load(f.read())
REQUIRED_FIELDS = ['name', 'doc']
+
def make_type(name, fields):
- return (collections.namedtuple(name, ' '.join(list(set(REQUIRED_FIELDS + fields)))), [])
+ return (collections.namedtuple(
+ name, ' '.join(list(set(REQUIRED_FIELDS + fields)))), [])
+
def c_str(s, encoding='ascii'):
- if isinstance(s, unicode):
- s = s.encode(encoding)
- result = ''
- for c in s:
- if not (32 <= ord(c) < 127) or c in ('\\', '"'):
- result += '\\%03o' % ord(c)
- else:
- result += c
- return '"' + result + '"'
-
-types = (
- make_type('Counter', []),
- make_type('Histogram', ['max', 'buckets']),
-)
+ if isinstance(s, unicode):
+ s = s.encode(encoding)
+ result = ''
+ for c in s:
+ if not (32 <= ord(c) < 127) or c in ('\\', '"'):
+ result += '\\%03o' % ord(c)
+ else:
+ result += c
+ return '"' + result + '"'
+
+
+types = (make_type('Counter', []), make_type('Histogram', ['max', 'buckets']),)
inst_map = dict((t[0].__name__, t[1]) for t in types)
stats = []
for attr in attrs:
- found = False
- for t, lst in types:
- t_name = t.__name__.lower()
- if t_name in attr:
- name = attr[t_name]
- del attr[t_name]
- lst.append(t(name=name, **attr))
- found = True
- break
- assert found, "Bad decl: %s" % attr
+ found = False
+ for t, lst in types:
+ t_name = t.__name__.lower()
+ if t_name in attr:
+ name = attr[t_name]
+ del attr[t_name]
+ lst.append(t(name=name, **attr))
+ found = True
+ break
+ assert found, "Bad decl: %s" % attr
+
def dbl2u64(d):
- return ctypes.c_ulonglong.from_buffer(ctypes.c_double(d)).value
+ return ctypes.c_ulonglong.from_buffer(ctypes.c_double(d)).value
+
def shift_works_until(mapped_bounds, shift_bits):
- for i, ab in enumerate(zip(mapped_bounds, mapped_bounds[1:])):
- a, b = ab
- if (a >> shift_bits) == (b >> shift_bits):
- return i
- return len(mapped_bounds)
+ for i, ab in enumerate(zip(mapped_bounds, mapped_bounds[1:])):
+ a, b = ab
+ if (a >> shift_bits) == (b >> shift_bits):
+ return i
+ return len(mapped_bounds)
+
def find_ideal_shift(mapped_bounds, max_size):
- best = None
- for shift_bits in reversed(range(0,64)):
- n = shift_works_until(mapped_bounds, shift_bits)
- if n == 0: continue
- table_size = mapped_bounds[n-1] >> shift_bits
- if table_size > max_size: continue
- if table_size > 65535: continue
- if best is None:
- best = (shift_bits, n, table_size)
- elif best[1] < n:
- best = (shift_bits, n, table_size)
- print best
- return best
+ best = None
+ for shift_bits in reversed(range(0, 64)):
+ n = shift_works_until(mapped_bounds, shift_bits)
+ if n == 0: continue
+ table_size = mapped_bounds[n - 1] >> shift_bits
+ if table_size > max_size: continue
+ if table_size > 65535: continue
+ if best is None:
+ best = (shift_bits, n, table_size)
+ elif best[1] < n:
+ best = (shift_bits, n, table_size)
+ print best
+ return best
+
def gen_map_table(mapped_bounds, shift_data):
- tbl = []
- cur = 0
- print mapped_bounds
- mapped_bounds = [x >> shift_data[0] for x in mapped_bounds]
- print mapped_bounds
- for i in range(0, mapped_bounds[shift_data[1]-1]):
- while i > mapped_bounds[cur]:
- cur += 1
- tbl.append(cur)
- return tbl
+ tbl = []
+ cur = 0
+ print mapped_bounds
+ mapped_bounds = [x >> shift_data[0] for x in mapped_bounds]
+ print mapped_bounds
+ for i in range(0, mapped_bounds[shift_data[1] - 1]):
+ while i > mapped_bounds[cur]:
+ cur += 1
+ tbl.append(cur)
+ return tbl
+
static_tables = []
+
def decl_static_table(values, type):
- global static_tables
- v = (type, values)
- for i, vp in enumerate(static_tables):
- if v == vp: return i
- print "ADD TABLE: %s %r" % (type, values)
- r = len(static_tables)
- static_tables.append(v)
- return r
+ global static_tables
+ v = (type, values)
+ for i, vp in enumerate(static_tables):
+ if v == vp: return i
+ print "ADD TABLE: %s %r" % (type, values)
+ r = len(static_tables)
+ static_tables.append(v)
+ return r
+
def type_for_uint_table(table):
- mv = max(table)
- if mv < 2**8:
- return 'uint8_t'
- elif mv < 2**16:
- return 'uint16_t'
- elif mv < 2**32:
- return 'uint32_t'
- else:
- return 'uint64_t'
+ mv = max(table)
+ if mv < 2**8:
+ return 'uint8_t'
+ elif mv < 2**16:
+ return 'uint16_t'
+ elif mv < 2**32:
+ return 'uint32_t'
+ else:
+ return 'uint64_t'
+
def gen_bucket_code(histogram):
- bounds = [0, 1]
- done_trivial = False
- done_unmapped = False
- first_nontrivial = None
- first_unmapped = None
- while len(bounds) < histogram.buckets + 1:
- if len(bounds) == histogram.buckets:
- nextb = int(histogram.max)
+ bounds = [0, 1]
+ done_trivial = False
+ done_unmapped = False
+ first_nontrivial = None
+ first_unmapped = None
+ while len(bounds) < histogram.buckets + 1:
+ if len(bounds) == histogram.buckets:
+ nextb = int(histogram.max)
+ else:
+ mul = math.pow(
+ float(histogram.max) / bounds[-1],
+ 1.0 / (histogram.buckets + 1 - len(bounds)))
+ nextb = int(math.ceil(bounds[-1] * mul))
+ if nextb <= bounds[-1] + 1:
+ nextb = bounds[-1] + 1
+ elif not done_trivial:
+ done_trivial = True
+ first_nontrivial = len(bounds)
+ bounds.append(nextb)
+ bounds_idx = decl_static_table(bounds, 'int')
+ if done_trivial:
+ first_nontrivial_code = dbl2u64(first_nontrivial)
+ code_bounds = [dbl2u64(x) - first_nontrivial_code for x in bounds]
+ shift_data = find_ideal_shift(code_bounds[first_nontrivial:],
+ 256 * histogram.buckets)
+ #print first_nontrivial, shift_data, bounds
+ #if shift_data is not None: print [hex(x >> shift_data[0]) for x in code_bounds[first_nontrivial:]]
+ code = 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max
+ map_table = gen_map_table(code_bounds[first_nontrivial:], shift_data)
+ if first_nontrivial is None:
+ code += ('GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, value);\n' %
+ histogram.name.upper())
else:
- mul = math.pow(float(histogram.max) / bounds[-1],
- 1.0 / (histogram.buckets + 1 - len(bounds)))
- nextb = int(math.ceil(bounds[-1] * mul))
- if nextb <= bounds[-1] + 1:
- nextb = bounds[-1] + 1
- elif not done_trivial:
- done_trivial = True
- first_nontrivial = len(bounds)
- bounds.append(nextb)
- bounds_idx = decl_static_table(bounds, 'int')
- if done_trivial:
- first_nontrivial_code = dbl2u64(first_nontrivial)
- code_bounds = [dbl2u64(x) - first_nontrivial_code for x in bounds]
- shift_data = find_ideal_shift(code_bounds[first_nontrivial:], 256 * histogram.buckets)
- #print first_nontrivial, shift_data, bounds
- #if shift_data is not None: print [hex(x >> shift_data[0]) for x in code_bounds[first_nontrivial:]]
- code = 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max
- map_table = gen_map_table(code_bounds[first_nontrivial:], shift_data)
- if first_nontrivial is None:
- code += ('GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, value);\n'
- % histogram.name.upper())
- else:
- code += 'if (value < %d) {\n' % first_nontrivial
- code += ('GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, value);\n'
- % histogram.name.upper())
- code += 'return;\n'
- code += '}'
- first_nontrivial_code = dbl2u64(first_nontrivial)
- if shift_data is not None:
- map_table_idx = decl_static_table(map_table, type_for_uint_table(map_table))
- code += 'union { double dbl; uint64_t uint; } _val, _bkt;\n'
- code += '_val.dbl = value;\n'
- code += 'if (_val.uint < %dull) {\n' % ((map_table[-1] << shift_data[0]) + first_nontrivial_code)
- code += 'int bucket = '
- code += 'grpc_stats_table_%d[((_val.uint - %dull) >> %d)] + %d;\n' % (map_table_idx, first_nontrivial_code, shift_data[0], first_nontrivial)
- code += '_bkt.dbl = grpc_stats_table_%d[bucket];\n' % bounds_idx
- code += 'bucket -= (_val.uint < _bkt.uint);\n'
- code += 'GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, bucket);\n' % histogram.name.upper()
- code += 'return;\n'
- code += '}\n'
- code += 'GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, '% histogram.name.upper()
- code += 'grpc_stats_histo_find_bucket_slow(value, grpc_stats_table_%d, %d));\n' % (bounds_idx, histogram.buckets)
- return (code, bounds_idx)
+ code += 'if (value < %d) {\n' % first_nontrivial
+ code += ('GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, value);\n' %
+ histogram.name.upper())
+ code += 'return;\n'
+ code += '}'
+ first_nontrivial_code = dbl2u64(first_nontrivial)
+ if shift_data is not None:
+ map_table_idx = decl_static_table(map_table,
+ type_for_uint_table(map_table))
+ code += 'union { double dbl; uint64_t uint; } _val, _bkt;\n'
+ code += '_val.dbl = value;\n'
+ code += 'if (_val.uint < %dull) {\n' % (
+ (map_table[-1] << shift_data[0]) + first_nontrivial_code)
+ code += 'int bucket = '
+ code += 'grpc_stats_table_%d[((_val.uint - %dull) >> %d)] + %d;\n' % (
+ map_table_idx, first_nontrivial_code, shift_data[0],
+ first_nontrivial)
+ code += '_bkt.dbl = grpc_stats_table_%d[bucket];\n' % bounds_idx
+ code += 'bucket -= (_val.uint < _bkt.uint);\n'
+ code += 'GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, bucket);\n' % histogram.name.upper(
+ )
+ code += 'return;\n'
+ code += '}\n'
+ code += 'GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, ' % histogram.name.upper(
+ )
+ code += 'grpc_stats_histo_find_bucket_slow(value, grpc_stats_table_%d, %d));\n' % (
+ bounds_idx, histogram.buckets)
+ return (code, bounds_idx)
+
# utility: print a big comment block into a set of files
def put_banner(files, banner):
- for f in files:
- print >>f, '/*'
- for line in banner:
- print >>f, ' * %s' % line
- print >>f, ' */'
- print >>f
+ for f in files:
+ print >> f, '/*'
+ for line in banner:
+ print >> f, ' * %s' % line
+ print >> f, ' */'
+ print >> f
+
with open('src/core/lib/debug/stats_data.h', 'w') as H:
- # copy-paste copyright notice from this file
- with open(sys.argv[0]) as my_source:
- copyright = []
- for line in my_source:
- if line[0] != '#': break
- for line in my_source:
- if line[0] == '#':
- copyright.append(line)
- break
- for line in my_source:
- if line[0] != '#':
- break
- copyright.append(line)
- put_banner([H], [line[2:].rstrip() for line in copyright])
-
- put_banner([H], ["Automatically generated by tools/codegen/core/gen_stats_data.py"])
-
- print >>H, "#ifndef GRPC_CORE_LIB_DEBUG_STATS_DATA_H"
- print >>H, "#define GRPC_CORE_LIB_DEBUG_STATS_DATA_H"
- print >>H
- print >>H, "#include <inttypes.h>"
- print >>H, "#include \"src/core/lib/iomgr/exec_ctx.h\""
- print >>H
- print >>H, "#ifdef __cplusplus"
- print >>H, "extern \"C\" {"
- print >>H, "#endif"
- print >>H
-
- for typename, instances in sorted(inst_map.items()):
- print >>H, "typedef enum {"
- for inst in instances:
- print >>H, " GRPC_STATS_%s_%s," % (typename.upper(), inst.name.upper())
- print >>H, " GRPC_STATS_%s_COUNT" % (typename.upper())
- print >>H, "} grpc_stats_%ss;" % (typename.lower())
- print >>H, "extern const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT];" % (
- typename.lower(), typename.upper())
- print >>H, "extern const char *grpc_stats_%s_doc[GRPC_STATS_%s_COUNT];" % (
- typename.lower(), typename.upper())
-
- histo_start = []
- histo_buckets = []
- histo_bucket_boundaries = []
-
- print >>H, "typedef enum {"
- first_slot = 0
- for histogram in inst_map['Histogram']:
- histo_start.append(first_slot)
- histo_buckets.append(histogram.buckets)
- print >>H, " GRPC_STATS_HISTOGRAM_%s_FIRST_SLOT = %d," % (histogram.name.upper(), first_slot)
- print >>H, " GRPC_STATS_HISTOGRAM_%s_BUCKETS = %d," % (histogram.name.upper(), histogram.buckets)
- first_slot += histogram.buckets
- print >>H, " GRPC_STATS_HISTOGRAM_BUCKETS = %d" % first_slot
- print >>H, "} grpc_stats_histogram_constants;"
-
- for ctr in inst_map['Counter']:
- print >>H, ("#define GRPC_STATS_INC_%s() " +
- "GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_%s)") % (
- ctr.name.upper(), ctr.name.upper())
- for histogram in inst_map['Histogram']:
- print >>H, "#define GRPC_STATS_INC_%s(value) grpc_stats_inc_%s( (int)(value))" % (
- histogram.name.upper(), histogram.name.lower())
- print >>H, "void grpc_stats_inc_%s(int x);" % histogram.name.lower()
-
- for i, tbl in enumerate(static_tables):
- print >>H, "extern const %s grpc_stats_table_%d[%d];" % (tbl[0], i, len(tbl[1]))
-
- print >>H, "extern const int grpc_stats_histo_buckets[%d];" % len(inst_map['Histogram'])
- print >>H, "extern const int grpc_stats_histo_start[%d];" % len(inst_map['Histogram'])
- print >>H, "extern const int *const grpc_stats_histo_bucket_boundaries[%d];" % len(inst_map['Histogram'])
- print >>H, "extern void (*const grpc_stats_inc_histogram[%d])(int x);" % len(inst_map['Histogram'])
-
- print >>H
- print >>H, "#ifdef __cplusplus"
- print >>H, "}"
- print >>H, "#endif"
- print >>H
- print >>H, "#endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */"
+ # copy-paste copyright notice from this file
+ with open(sys.argv[0]) as my_source:
+ copyright = []
+ for line in my_source:
+ if line[0] != '#': break
+ for line in my_source:
+ if line[0] == '#':
+ copyright.append(line)
+ break
+ for line in my_source:
+ if line[0] != '#':
+ break
+ copyright.append(line)
+ put_banner([H], [line[2:].rstrip() for line in copyright])
+
+ put_banner(
+ [H],
+ ["Automatically generated by tools/codegen/core/gen_stats_data.py"])
+
+ print >> H, "#ifndef GRPC_CORE_LIB_DEBUG_STATS_DATA_H"
+ print >> H, "#define GRPC_CORE_LIB_DEBUG_STATS_DATA_H"
+ print >> H
+ print >> H, "#include <inttypes.h>"
+ print >> H, "#include \"src/core/lib/iomgr/exec_ctx.h\""
+ print >> H
+ print >> H, "#ifdef __cplusplus"
+ print >> H, "extern \"C\" {"
+ print >> H, "#endif"
+ print >> H
+
+ for typename, instances in sorted(inst_map.items()):
+ print >> H, "typedef enum {"
+ for inst in instances:
+ print >> H, " GRPC_STATS_%s_%s," % (typename.upper(),
+ inst.name.upper())
+ print >> H, " GRPC_STATS_%s_COUNT" % (typename.upper())
+ print >> H, "} grpc_stats_%ss;" % (typename.lower())
+ print >> H, "extern const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT];" % (
+ typename.lower(), typename.upper())
+ print >> H, "extern const char *grpc_stats_%s_doc[GRPC_STATS_%s_COUNT];" % (
+ typename.lower(), typename.upper())
+
+ histo_start = []
+ histo_buckets = []
+ histo_bucket_boundaries = []
+
+ print >> H, "typedef enum {"
+ first_slot = 0
+ for histogram in inst_map['Histogram']:
+ histo_start.append(first_slot)
+ histo_buckets.append(histogram.buckets)
+ print >> H, " GRPC_STATS_HISTOGRAM_%s_FIRST_SLOT = %d," % (
+ histogram.name.upper(), first_slot)
+ print >> H, " GRPC_STATS_HISTOGRAM_%s_BUCKETS = %d," % (
+ histogram.name.upper(), histogram.buckets)
+ first_slot += histogram.buckets
+ print >> H, " GRPC_STATS_HISTOGRAM_BUCKETS = %d" % first_slot
+ print >> H, "} grpc_stats_histogram_constants;"
+
+ for ctr in inst_map['Counter']:
+ print >> H, ("#define GRPC_STATS_INC_%s() " +
+ "GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_%s)") % (
+ ctr.name.upper(), ctr.name.upper())
+ for histogram in inst_map['Histogram']:
+ print >> H, "#define GRPC_STATS_INC_%s(value) grpc_stats_inc_%s( (int)(value))" % (
+ histogram.name.upper(), histogram.name.lower())
+ print >> H, "void grpc_stats_inc_%s(int x);" % histogram.name.lower()
+
+ for i, tbl in enumerate(static_tables):
+ print >> H, "extern const %s grpc_stats_table_%d[%d];" % (tbl[0], i,
+ len(tbl[1]))
+
+ print >> H, "extern const int grpc_stats_histo_buckets[%d];" % len(
+ inst_map['Histogram'])
+ print >> H, "extern const int grpc_stats_histo_start[%d];" % len(
+ inst_map['Histogram'])
+ print >> H, "extern const int *const grpc_stats_histo_bucket_boundaries[%d];" % len(
+ inst_map['Histogram'])
+ print >> H, "extern void (*const grpc_stats_inc_histogram[%d])(int x);" % len(
+ inst_map['Histogram'])
+
+ print >> H
+ print >> H, "#ifdef __cplusplus"
+ print >> H, "}"
+ print >> H, "#endif"
+ print >> H
+ print >> H, "#endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */"
with open('src/core/lib/debug/stats_data.cc', 'w') as C:
- # copy-paste copyright notice from this file
- with open(sys.argv[0]) as my_source:
- copyright = []
- for line in my_source:
- if line[0] != '#': break
- for line in my_source:
- if line[0] == '#':
- copyright.append(line)
- break
- for line in my_source:
- if line[0] != '#':
- break
- copyright.append(line)
- put_banner([C], [line[2:].rstrip() for line in copyright])
-
- put_banner([C], ["Automatically generated by tools/codegen/core/gen_stats_data.py"])
-
- print >>C, "#include \"src/core/lib/debug/stats_data.h\""
- print >>C, "#include \"src/core/lib/debug/stats.h\""
- print >>C, "#include \"src/core/lib/iomgr/exec_ctx.h\""
- print >>C, "#include <grpc/support/useful.h>"
-
- histo_code = []
- for histogram in inst_map['Histogram']:
- code, bounds_idx = gen_bucket_code(histogram)
- histo_bucket_boundaries.append(bounds_idx)
- histo_code.append(code)
-
- for typename, instances in sorted(inst_map.items()):
- print >>C, "const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT] = {" % (
- typename.lower(), typename.upper())
- for inst in instances:
- print >>C, " %s," % c_str(inst.name)
- print >>C, "};"
- print >>C, "const char *grpc_stats_%s_doc[GRPC_STATS_%s_COUNT] = {" % (
- typename.lower(), typename.upper())
- for inst in instances:
- print >>C, " %s," % c_str(inst.doc)
- print >>C, "};"
-
- for i, tbl in enumerate(static_tables):
- print >>C, "const %s grpc_stats_table_%d[%d] = {%s};" % (
- tbl[0], i, len(tbl[1]), ','.join('%s' % x for x in tbl[1]))
-
- for histogram, code in zip(inst_map['Histogram'], histo_code):
- print >>C, ("void grpc_stats_inc_%s(int value) {%s}") % (
- histogram.name.lower(),
- code)
-
- print >>C, "const int grpc_stats_histo_buckets[%d] = {%s};" % (
- len(inst_map['Histogram']), ','.join('%s' % x for x in histo_buckets))
- print >>C, "const int grpc_stats_histo_start[%d] = {%s};" % (
- len(inst_map['Histogram']), ','.join('%s' % x for x in histo_start))
- print >>C, "const int *const grpc_stats_histo_bucket_boundaries[%d] = {%s};" % (
- len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x for x in histo_bucket_boundaries))
- print >>C, "void (*const grpc_stats_inc_histogram[%d])(int x) = {%s};" % (
- len(inst_map['Histogram']), ','.join('grpc_stats_inc_%s' % histogram.name.lower() for histogram in inst_map['Histogram']))
+ # copy-paste copyright notice from this file
+ with open(sys.argv[0]) as my_source:
+ copyright = []
+ for line in my_source:
+ if line[0] != '#': break
+ for line in my_source:
+ if line[0] == '#':
+ copyright.append(line)
+ break
+ for line in my_source:
+ if line[0] != '#':
+ break
+ copyright.append(line)
+ put_banner([C], [line[2:].rstrip() for line in copyright])
+
+ put_banner(
+ [C],
+ ["Automatically generated by tools/codegen/core/gen_stats_data.py"])
+
+ print >> C, "#include \"src/core/lib/debug/stats_data.h\""
+ print >> C, "#include \"src/core/lib/debug/stats.h\""
+ print >> C, "#include \"src/core/lib/iomgr/exec_ctx.h\""
+ print >> C, "#include <grpc/support/useful.h>"
+
+ histo_code = []
+ for histogram in inst_map['Histogram']:
+ code, bounds_idx = gen_bucket_code(histogram)
+ histo_bucket_boundaries.append(bounds_idx)
+ histo_code.append(code)
+
+ for typename, instances in sorted(inst_map.items()):
+ print >> C, "const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT] = {" % (
+ typename.lower(), typename.upper())
+ for inst in instances:
+ print >> C, " %s," % c_str(inst.name)
+ print >> C, "};"
+ print >> C, "const char *grpc_stats_%s_doc[GRPC_STATS_%s_COUNT] = {" % (
+ typename.lower(), typename.upper())
+ for inst in instances:
+ print >> C, " %s," % c_str(inst.doc)
+ print >> C, "};"
+
+ for i, tbl in enumerate(static_tables):
+ print >> C, "const %s grpc_stats_table_%d[%d] = {%s};" % (
+ tbl[0], i, len(tbl[1]), ','.join('%s' % x for x in tbl[1]))
+
+ for histogram, code in zip(inst_map['Histogram'], histo_code):
+ print >> C, ("void grpc_stats_inc_%s(int value) {%s}") % (
+ histogram.name.lower(), code)
+
+ print >> C, "const int grpc_stats_histo_buckets[%d] = {%s};" % (
+ len(inst_map['Histogram']), ','.join('%s' % x for x in histo_buckets))
+ print >> C, "const int grpc_stats_histo_start[%d] = {%s};" % (
+ len(inst_map['Histogram']), ','.join('%s' % x for x in histo_start))
+ print >> C, "const int *const grpc_stats_histo_bucket_boundaries[%d] = {%s};" % (
+ len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x
+ for x in histo_bucket_boundaries))
+ print >> C, "void (*const grpc_stats_inc_histogram[%d])(int x) = {%s};" % (
+ len(inst_map['Histogram']), ','.join(
+ 'grpc_stats_inc_%s' % histogram.name.lower()
+ for histogram in inst_map['Histogram']))
# patch qps_test bigquery schema
RECORD_EXPLICIT_PERCENTILES = [50, 95, 99]
with open('tools/run_tests/performance/scenario_result_schema.json', 'r') as f:
- qps_schema = json.loads(f.read())
+ qps_schema = json.loads(f.read())
+
def FindNamed(js, name):
- for el in js:
- if el['name'] == name:
- return el
+ for el in js:
+ if el['name'] == name:
+ return el
+
def RemoveCoreFields(js):
- new_fields = []
- for field in js['fields']:
- if not field['name'].startswith('core_'):
- new_fields.append(field)
- js['fields'] = new_fields
+ new_fields = []
+ for field in js['fields']:
+ if not field['name'].startswith('core_'):
+ new_fields.append(field)
+ js['fields'] = new_fields
+
RemoveCoreFields(FindNamed(qps_schema, 'clientStats'))
RemoveCoreFields(FindNamed(qps_schema, 'serverStats'))
+
def AddCoreFields(js):
- for counter in inst_map['Counter']:
- js['fields'].append({
- 'name': 'core_%s' % counter.name,
- 'type': 'INTEGER',
- 'mode': 'NULLABLE'
- })
- for histogram in inst_map['Histogram']:
- js['fields'].append({
- 'name': 'core_%s' % histogram.name,
- 'type': 'STRING',
- 'mode': 'NULLABLE'
- })
- js['fields'].append({
- 'name': 'core_%s_bkts' % histogram.name,
- 'type': 'STRING',
- 'mode': 'NULLABLE'
- })
- for pctl in RECORD_EXPLICIT_PERCENTILES:
- js['fields'].append({
- 'name': 'core_%s_%dp' % (histogram.name, pctl),
- 'type': 'FLOAT',
- 'mode': 'NULLABLE'
- })
+ for counter in inst_map['Counter']:
+ js['fields'].append({
+ 'name': 'core_%s' % counter.name,
+ 'type': 'INTEGER',
+ 'mode': 'NULLABLE'
+ })
+ for histogram in inst_map['Histogram']:
+ js['fields'].append({
+ 'name': 'core_%s' % histogram.name,
+ 'type': 'STRING',
+ 'mode': 'NULLABLE'
+ })
+ js['fields'].append({
+ 'name': 'core_%s_bkts' % histogram.name,
+ 'type': 'STRING',
+ 'mode': 'NULLABLE'
+ })
+ for pctl in RECORD_EXPLICIT_PERCENTILES:
+ js['fields'].append({
+ 'name': 'core_%s_%dp' % (histogram.name, pctl),
+ 'type': 'FLOAT',
+ 'mode': 'NULLABLE'
+ })
+
AddCoreFields(FindNamed(qps_schema, 'clientStats'))
AddCoreFields(FindNamed(qps_schema, 'serverStats'))
with open('tools/run_tests/performance/scenario_result_schema.json', 'w') as f:
- f.write(json.dumps(qps_schema, indent=2, sort_keys=True))
+ f.write(json.dumps(qps_schema, indent=2, sort_keys=True))
# and generate a helper script to massage scenario results into the format we'd
# like to query
with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P:
- with open(sys.argv[0]) as my_source:
- for line in my_source:
- if line[0] != '#': break
- for line in my_source:
- if line[0] == '#':
- print >>P, line.rstrip()
- break
- for line in my_source:
- if line[0] != '#':
- break
- print >>P, line.rstrip()
-
- print >>P
- print >>P, '# Autogenerated by tools/codegen/core/gen_stats_data.py'
- print >>P
-
- print >>P, 'import massage_qps_stats_helpers'
-
- print >>P, 'def massage_qps_stats(scenario_result):'
- print >>P, ' for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:'
- print >>P, ' if "coreStats" not in stats: return'
- print >>P, ' core_stats = stats["coreStats"]'
- print >>P, ' del stats["coreStats"]'
- for counter in inst_map['Counter']:
- print >>P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (counter.name, counter.name)
- for i, histogram in enumerate(inst_map['Histogram']):
- print >>P, ' h = massage_qps_stats_helpers.histogram(core_stats, "%s")' % histogram.name
- print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in h.buckets)' % histogram.name
- print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in h.boundaries)' % histogram.name
- for pctl in RECORD_EXPLICIT_PERCENTILES:
- print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(h.buckets, %d, h.boundaries)' % (
- histogram.name, pctl, pctl)
+ with open(sys.argv[0]) as my_source:
+ for line in my_source:
+ if line[0] != '#': break
+ for line in my_source:
+ if line[0] == '#':
+ print >> P, line.rstrip()
+ break
+ for line in my_source:
+ if line[0] != '#':
+ break
+ print >> P, line.rstrip()
+
+ print >> P
+ print >> P, '# Autogenerated by tools/codegen/core/gen_stats_data.py'
+ print >> P
+
+ print >> P, 'import massage_qps_stats_helpers'
+
+ print >> P, 'def massage_qps_stats(scenario_result):'
+ print >> P, ' for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:'
+ print >> P, ' if "coreStats" not in stats: return'
+ print >> P, ' core_stats = stats["coreStats"]'
+ print >> P, ' del stats["coreStats"]'
+ for counter in inst_map['Counter']:
+ print >> P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (
+ counter.name, counter.name)
+ for i, histogram in enumerate(inst_map['Histogram']):
+ print >> P, ' h = massage_qps_stats_helpers.histogram(core_stats, "%s")' % histogram.name
+ print >> P, ' stats["core_%s"] = ",".join("%%f" %% x for x in h.buckets)' % histogram.name
+ print >> P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in h.boundaries)' % histogram.name
+ for pctl in RECORD_EXPLICIT_PERCENTILES:
+ print >> P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(h.buckets, %d, h.boundaries)' % (
+ histogram.name, pctl, pctl)
with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S:
- columns = []
- for counter in inst_map['Counter']:
- columns.append(('%s_per_iteration' % counter.name, 'FLOAT'))
- print >>S, ',\n'.join('%s:%s' % x for x in columns)
-
+ columns = []
+ for counter in inst_map['Counter']:
+ columns.append(('%s_per_iteration' % counter.name, 'FLOAT'))
+ print >> S, ',\n'.join('%s:%s' % x for x in columns)