Switches to a Skia-specific appengine entry point that uses condensed data and taskqueue writes (SkipBuildbotRuns).

The default entry /add_point does not handle large data efficiently, so we've seen >30min timeouts for some bots to upload data. We now switch to using /skia_add_points that I'm writing for Skia, so we can condense the data to send to minimum (since all points in each batch have the same revision, platform and config), and dedicate the actual data processing to /skia_write_datastore which is a taskqueue task that has longer timeout and can run at app backend instead of blocking the bots.
Initial testing from my MacBook on Mac 64 bench data via vpn gave only a little more than 1 second for uploading data for one config, about 15 seconds for all 12 configs. That's a big win against 20+ minutes.
Review URL: https://codereview.chromium.org/13762002

git-svn-id: http://skia.googlecode.com/svn/trunk@8560 2bbb7eff-a529-9590-31e7-b0007b416f81

1 files changed, 23 insertions, 29 deletions

diff --git a/bench/bench_graph_svg.py b/bench/bench_graph_svg.py
index 7ac6237697..57be7ea4ca 100644
--- a/bench/bench_graph_svg.py
+++ b/bench/bench_graph_svg.py
@@ -23,17 +23,6 @@ MAX_REASONABLE_TIME = 99999
 TITLE_PREAMBLE = 'Bench_Performance_for_Skia_'
 TITLE_PREAMBLE_LENGTH = len(TITLE_PREAMBLE)
 
-# Number of data points to send to appengine at once.
-DATA_POINT_BATCHSIZE = 66
-
-def grouper(n, iterable):
-    """Groups list into list of lists for a given size. See itertools doc:
-      http://docs.python.org/2/library/itertools.html#module-itertools
-    """
-    args = [iter(iterable)] * n
-    return [[n for n in t if n] for t in itertools.izip_longest(*args)]
-
-
 def usage():
     """Prints simple usage information."""
 
@@ -413,7 +402,7 @@ def main():
           newest_revision: the latest revision that this script reads
           bot: the bot platform the bench is run on
         """
-        data = []
+        config_data_dic = {}
         for label in line_data_dict.iterkeys():
             if not label.bench.endswith('.skp') or label.time_type:
                 # filter out non-picture and non-walltime benches
@@ -424,23 +413,28 @@ def main():
             # data point we have for each line.
             if rev != newest_revision:
                 continue
-            data.append({'master': 'Skia', 'bot': bot,
-                         'test': config + '/' + label.bench.replace('.skp', ''),
-                         'revision': rev, 'value': val, 'error': 0})
-        for curr_data in grouper(DATA_POINT_BATCHSIZE, data):
-            req = urllib2.Request(appengine_url,
-                urllib.urlencode({'data': json.dumps(curr_data)}))
-            try:
-              urllib2.urlopen(req)
-            except urllib2.HTTPError, e:
-                sys.stderr.write("HTTPError for JSON data %s: %s\n" % (
-                    data, e))
-            except urllib2.URLError, e:
-                sys.stderr.write("URLError for JSON data %s: %s\n" % (
-                    data, e))
-            except httplib.HTTPException, e:
-                sys.stderr.write("HTTPException for JSON data %s: %s\n" % (
-                    data, e))
+            if config not in config_data_dic:
+                config_data_dic[config] = []
+            config_data_dic[config].append(label.bench.replace('.skp', '') +
+                ':%.2f' % val)
+        for config in config_data_dic:
+            if config_data_dic[config]:
+                data = {'master': 'Skia', 'bot': bot, 'test': config,
+                        'revision': newest_revision,
+                        'benches': ','.join(config_data_dic[config])}
+                req = urllib2.Request(appengine_url,
+                    urllib.urlencode({'data': json.dumps(data)}))
+                try:
+                    urllib2.urlopen(req)
+                except urllib2.HTTPError, e:
+                    sys.stderr.write("HTTPError for JSON data %s: %s\n" % (
+                        data, e))
+                except urllib2.URLError, e:
+                    sys.stderr.write("URLError for JSON data %s: %s\n" % (
+                        data, e))
+                except httplib.HTTPException, e:
+                    sys.stderr.write("HTTPException for JSON data %s: %s\n" % (
+                        data, e))
 
     try:
         for option, value in opts: