diff options
| -rw-r--r-- | Makefile | 39 | ||||
| -rw-r--r-- | build.json | 6 | ||||
| -rw-r--r-- | test/cpp/qps/report.cc | 73 | ||||
| -rw-r--r-- | test/cpp/qps/report.h | 18 | ||||
| -rwxr-xr-x | test/cpp/qps/run_perf_db_test.py | 273 | ||||
| -rw-r--r-- | test/cpp/qps/user_data.proto | 122 | ||||
| -rw-r--r-- | test/cpp/qps/user_data_client.cc | 114 | ||||
| -rw-r--r-- | test/cpp/qps/user_data_client.h | 100 | ||||
| -rw-r--r-- | test/cpp/util/benchmark_config.cc | 13 |
9 files changed, 748 insertions, 10 deletions
@@ -2517,6 +2517,21 @@ $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc: test/cpp/qps/qpstest.proto $(PROTOBUF endif ifeq ($(NO_PROTOC),true) +$(GENDIR)/test/cpp/qps/user_data.pb.cc: protoc_dep_error +$(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc: protoc_dep_error +else +$(GENDIR)/test/cpp/qps/user_data.pb.cc: test/cpp/qps/user_data.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) + $(E) "[PROTOC] Generating protobuf CC file from $<" + $(Q) mkdir -p `dirname $@` + $(Q) $(PROTOC) --cpp_out=$(GENDIR) $< + +$(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc: test/cpp/qps/user_data.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) + $(E) "[GRPC] Generating gRPC's protobuf service CC file from $<" + $(Q) mkdir -p `dirname $@` + $(Q) $(PROTOC) --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $< +endif + +ifeq ($(NO_PROTOC),true) $(GENDIR)/test/cpp/util/echo.pb.cc: protoc_dep_error $(GENDIR)/test/cpp/util/echo.grpc.pb.cc: protoc_dep_error else @@ -3513,7 +3528,9 @@ endif LIBGRPC++_BENCHMARK_CONFIG_SRC = \ $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc \ + $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc \ test/cpp/qps/report.cc \ + test/cpp/qps/user_data_client.cc \ test/cpp/util/benchmark_config.cc \ @@ -3558,8 +3575,9 @@ ifneq ($(NO_DEPS),true) -include $(LIBGRPC++_BENCHMARK_CONFIG_OBJS:.o=.dep) endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/qps/report.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/util/benchmark_config.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/report.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/user_data_client.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/util/benchmark_config.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc LIBGRPC++_TEST_CONFIG_SRC = \ @@ -4072,6 +4090,7 @@ $(OBJDIR)/$(CONFIG)/examples/pubsub/subscriber.o: $(GENDIR)/examples/pubsub/labe LIBQPS_SRC = \ $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc \ + $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc \ test/cpp/qps/client_async.cc \ test/cpp/qps/client_sync.cc \ test/cpp/qps/driver.cc \ @@ -4079,6 +4098,7 @@ LIBQPS_SRC = \ test/cpp/qps/server_async.cc \ test/cpp/qps/server_sync.cc \ test/cpp/qps/timer.cc \ + test/cpp/qps/user_data_client.cc \ LIBQPS_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBQPS_SRC)))) @@ -4122,13 +4142,14 @@ ifneq ($(NO_DEPS),true) -include $(LIBQPS_OBJS:.o=.dep) endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/qps/client_async.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/client_sync.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/driver.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_worker.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/server_async.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/server_sync.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/timer.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/client_async.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/client_sync.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/driver.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_worker.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/server_async.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/server_sync.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/timer.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/user_data_client.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc LIBGRPC_CSHARP_EXT_SRC = \ diff --git a/build.json b/build.json index d93254300c..c68e1d52df 100644 --- a/build.json +++ b/build.json @@ -539,7 +539,9 @@ "language": "c++", "src": [ "test/cpp/qps/qpstest.proto", + "test/cpp/qps/user_data.proto", "test/cpp/qps/report.cc", + "test/cpp/qps/user_data_client.cc", "test/cpp/util/benchmark_config.cc" ] }, @@ -710,13 +712,15 @@ ], "src": [ "test/cpp/qps/qpstest.proto", + "test/cpp/qps/user_data.proto", "test/cpp/qps/client_async.cc", "test/cpp/qps/client_sync.cc", "test/cpp/qps/driver.cc", "test/cpp/qps/qps_worker.cc", "test/cpp/qps/server_async.cc", "test/cpp/qps/server_sync.cc", - "test/cpp/qps/timer.cc" + "test/cpp/qps/timer.cc", + "test/cpp/qps/user_data_client.cc" ], "deps": [ "grpc_test_util", diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index 678ea080d1..c524327ff0 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -35,6 +35,7 @@ #include <grpc/support/log.h> #include "test/cpp/qps/stats.h" +#include "user_data_client.h" namespace grpc { namespace testing { @@ -118,5 +119,77 @@ void GprLogReporter::ReportTimes(const ScenarioResult& result) const { [](ResourceUsage u) { return u.wall_time; })); } +UserDataClient userDataClient(grpc::CreateChannel("localhost:50052", grpc::InsecureCredentials(), + ChannelArguments())); + +//Performance database reporter implementation. +void UserDatabaseReporter::ReportQPS(const ScenarioResult& result) const { + auto qps = result.latencies.Count() / + average(result.client_resources, + [](ResourceUsage u) { return u.wall_time; }); + + userDataClient.setQPS(qps); + userDataClient.setConfigs(result.client_config, result.server_config); +} + +void UserDatabaseReporter::ReportQPSPerCore(const ScenarioResult& result) const { + auto qps = result.latencies.Count() / + average(result.client_resources, + [](ResourceUsage u) { return u.wall_time; }); + + auto qpsPerCore = qps / result.server_config.threads(); + + userDataClient.setQPS(qps); + userDataClient.setQPSPerCore(qpsPerCore); + userDataClient.setConfigs(result.client_config, result.server_config); +} + +void UserDatabaseReporter::ReportLatency(const ScenarioResult& result) const { + userDataClient.setLatencies(result.latencies.Percentile(50) / 1000, + result.latencies.Percentile(90) / 1000, + result.latencies.Percentile(95) / 1000, + result.latencies.Percentile(99) / 1000, + result.latencies.Percentile(99.9) / 1000); + userDataClient.setConfigs(result.client_config, result.server_config); +} + +void UserDatabaseReporter::ReportTimes(const ScenarioResult& result) const { + double serverSystemTime = 100.0 * sum(result.server_resources, + [](ResourceUsage u) { return u.system_time; }) / + sum(result.server_resources, + [](ResourceUsage u) { return u.wall_time; }); + double serverUserTime = 100.0 * sum(result.server_resources, + [](ResourceUsage u) { return u.user_time; }) / + sum(result.server_resources, + [](ResourceUsage u) { return u.wall_time; }); + double clientSystemTime = 100.0 * sum(result.client_resources, + [](ResourceUsage u) { return u.system_time; }) / + sum(result.client_resources, + [](ResourceUsage u) { return u.wall_time; }); + double clientUserTime = 100.0 * sum(result.client_resources, + [](ResourceUsage u) { return u.user_time; }) / + sum(result.client_resources, + [](ResourceUsage u) { return u.wall_time; }); + + userDataClient.setTimes(serverSystemTime, serverUserTime, + clientSystemTime, clientUserTime); + userDataClient.setConfigs(result.client_config, result.server_config); +} + +void UserDatabaseReporter::SendData() const { + //send data to performance database + int userDataState = userDataClient.sendData(access_token_, test_name_, sys_info_); + + //check state of data sending + switch(userDataState) { + case 1: + gpr_log(GPR_INFO, "Data sent to user database successfully"); + break; + case -1: + gpr_log(GPR_INFO, "Data could not be sent to user database"); + break; + } +} + } // namespace testing } // namespace grpc diff --git a/test/cpp/qps/report.h b/test/cpp/qps/report.h index 0cce08816a..432ed5ddf2 100644 --- a/test/cpp/qps/report.h +++ b/test/cpp/qps/report.h @@ -103,6 +103,24 @@ class GprLogReporter : public Reporter { void ReportTimes(const ScenarioResult& result) const GRPC_OVERRIDE; }; +/** Reporter for client leaderboard. */ +class UserDatabaseReporter : public Reporter { + public: + UserDatabaseReporter(const string& name, const string& access_token, const string& test_name, const string& sys_info) + : Reporter(name), access_token_(access_token), test_name_(test_name), sys_info_(sys_info) {} + ~UserDatabaseReporter() { SendData(); }; + + private: + std::string access_token_; + std::string test_name_; + std::string sys_info_; + void ReportQPS(const ScenarioResult& result) const GRPC_OVERRIDE; + void ReportQPSPerCore(const ScenarioResult& result) const GRPC_OVERRIDE; + void ReportLatency(const ScenarioResult& result) const GRPC_OVERRIDE; + void ReportTimes(const ScenarioResult& result) const GRPC_OVERRIDE; + void SendData() const; +}; + } // namespace testing } // namespace grpc diff --git a/test/cpp/qps/run_perf_db_test.py b/test/cpp/qps/run_perf_db_test.py new file mode 100755 index 0000000000..7f840fa723 --- /dev/null +++ b/test/cpp/qps/run_perf_db_test.py @@ -0,0 +1,273 @@ +#!/usr/bin/python +# +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +import os +import sys +import re +import urllib2 +import urllib +import json +import time +import subprocess +import fnmatch + +CLIENT_ID = '1018396037782-tv81fshn76nemr24uuhuginceb9hni2m.apps.googleusercontent.com' +CLIENT_SECRET = '_HGHXg4DAA59r4w4x8p6ARzD' +GRANT_TYPE = 'http://oauth.net/grant_type/device/1.0' +ACCESS_TOKENS_DIR = '/usr/local/auth_access_tokens' +AUTH_TOKEN_LINK = 'https://www.googleapis.com/oauth2/v3/token' +GOOGLE_ACCOUNTS_LINK = 'https://accounts.google.com/o/oauth2/device/code' +USER_INFO_LINK = 'https://www.googleapis.com/oauth2/v1/userinfo' + +# Fetches JSON reply object, given a url and parameters +def fetchJSON(url, paramDict): + if len(paramDict) == 0: + req = urllib2.Request(url) + else: + data = urllib.urlencode(paramDict) + req = urllib2.Request(url, data) + + try: + response = urllib2.urlopen(req) + result = response.read() + + except urllib2.HTTPError, error: + result = error.read() + + return result + +# Fetch user info; used to check if access token is valid +def getUserInfo(accessToken): + url = USER_INFO_LINK + '?access_token=' + accessToken + paramDict = {} + JSONBody = fetchJSON(url, paramDict) + data = json.loads(JSONBody) + + return data + +# Returns true if stored access token is valid +def isAccessTokenValid(accessToken): + data = getUserInfo(accessToken); + + if 'id' in data: + return True + else: + return False + +# Returns user id given a working access token +def getUserId(accessToken): + data = getUserInfo(accessToken) + + email = data['email'] + userId = getUserIdFromEmail(email) + + return userId + +# Extracts a unique user id from an email address +def getUserIdFromEmail(email): + email = email.split('@')[0].lower() # take username and convert to lower case + userId = re.sub('[.]', '', email) # remove periods + + return userId + +# Use an existing access token +def useAccessToken(userTokFile): + with open(userTokFile, "r") as data_file: + data = json.load(data_file) # load JSON data from file + accessToken = data["access_token"] + + # If access token has gone stale, refresh it + if not isAccessTokenValid(accessToken): + return refreshAccessToken(data["refresh_token"], userTokFile) + + return accessToken + +# refresh stale access token +def refreshAccessToken(refreshToken, userTokFile): + # Parameters for request + paramDict = {'refresh_token':refreshToken, 'client_id':CLIENT_ID, 'client_secret':CLIENT_SECRET, 'grant_type':'refresh_token'} + # Fetch reply to request + JSONBody = fetchJSON(AUTH_TOKEN_LINK, paramDict) + data = json.loads(JSONBody) + + if not 'access_token' in data: + # Refresh token has gone stale, re-authentication required + return reauthenticate() + else: + # write fresh access token to tokens file + tokenData = {} + + with open(userTokFile, "r") as data_file: + tokenData = json.load(data_file) + + with open(userTokFile, "w") as data_file: + tokenData['access_token'] = data['access_token'] + json.dump(tokenData, data_file) + + # return fresh access token + return data['access_token'] + +def reauthenticate(): + # Request parameters + paramDict = {'client_id':CLIENT_ID, 'scope':'email profile'} + JSONBody = fetchJSON(GOOGLE_ACCOUNTS_LINK, paramDict) + data = json.loads(JSONBody) + + print 'User authorization required\n' + print 'Please use the following code in you browser: ', data['user_code'] # Code to be entered by user in browser + print 'Verification URL: ', data['verification_url'] # Authentication link + print '\nAwaiting user authorization. May take a few more seconds after authorizing...\n' + + authData = {} + + while not 'access_token' in authData: + # Request parameters + authDict = {'client_id':CLIENT_ID, 'client_secret':CLIENT_SECRET, 'code':data['device_code'], 'grant_type':GRANT_TYPE} + JSONBody = fetchJSON(AUTH_TOKEN_LINK, authDict) + authData = json.loads(JSONBody) + # If server pinged too quickly, will get slowdown message; need to wait for specified interval + time.sleep(data['interval']) + + # File to write tokens + newUserTokFile = ACCESS_TOKENS_DIR + '/' + getUserId(authData['access_token']) + + # Write tokens to file + with open(newUserTokFile, "w") as data_file: + json.dump(authData, data_file) + + # return working access token + return authData['access_token'] + +# Fetch a working access token given user entered email id; authntication may be required +def getAccessToken(email): + # Get unique user id from email address + userId = getUserIdFromEmail(email) + + # Token file + userTokFile = ACCESS_TOKENS_DIR + '/' + userId + + accessToken = '' + + if os.path.exists(userTokFile): + # File containing access token exists; unless refresh token has expired, user authentication will not be required + accessToken = useAccessToken(userTokFile) + else: + # User authentication required + accessToken = reauthenticate() + + return accessToken + +# If user has not entered full path to test, recursively searches for given test in parent folders +def findTestPath(test): + # If user entered full path to test, return it + if(os.path.isfile(test)): + return test + + testName = test.split('/')[-1] # Extract just test name + testPath = '' + + # Search for test + for root, dirnames, filenames in os.walk('../../../'): + for fileName in fnmatch.filter(filenames, testName): + testPath = os.path.join(root, fileName) + + return testPath + +def getSysInfo(): + # Fetch system information + sysInfo = os.popen('lscpu').readlines() + + NICs = os.popen('ifconfig | cut -c1-8 | sed \'/^\s*$/d\' | sort -u').readlines() + nicAddrs = os.popen('ifconfig | grep -oE "inet addr:([0-9]{1,3}\.){3}[0-9]{1,3}"').readlines() + + nicInfo = [] + + for i in range(0, len(NICs)): + NIC = NICs[i] + NIC = re.sub(r'[^\w]', '', NIC) + + ethtoolProcess = subprocess.Popen(["ethtool",NIC], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + ethtoolResult = ethtoolProcess.communicate()[0] + + ethtoolResultList = ethtoolResult.split('\n\t') + for ethtoolString in ethtoolResultList: + if ethtoolString.startswith('Speed'): + ethtoolString = ethtoolString.split(':')[1] + ethtoolString = ethtoolString.replace('Mb/s',' Mbps') + nicInfo.append('NIC ' + NIC + ' speed: ' + ethtoolString + '\n') + nicInfo.append(NIC + ' inet address: ' + nicAddrs[i].split(':')[1]) + + print 'Obtaining network info....' + tcp_rr_rate = str(os.popen('netperf -t TCP_RR -v 0').readlines()[1]) + print 'Network info obtained' + + nicInfo.append('TCP RR Transmission Rate per sec: ' + tcp_rr_rate + '\n') + sysInfo = sysInfo + nicInfo + + return sysInfo + +def main(): + # If tokens directory does not exist, creates it + if not os.path.exists(ACCESS_TOKENS_DIR): + subprocess.call(['sudo', 'mkdir', ACCESS_TOKENS_DIR]) + subprocess.call(['sudo', 'chmod', '777', ACCESS_TOKENS_DIR]) + + if len(sys.argv) > 1: + test = sys.argv[1] + else: + test = raw_input('Enter the test path/name: ') + + if len(sys.argv) > 2: + email = sys.argv[2] + else: + email = raw_input('Enter your e-mail id: ') + + try: + # Fetch working access token + accessToken = getAccessToken(email) + except Exception, e: + print 'Error in authentication' + + try: + testPath = findTestPath(test) # Get path to test + testName = testPath.split('/')[-1] # Get test name + + sysInfo = getSysInfo() + + print '\nBeginning test:\n' + # Run the test + subprocess.call([testPath, '--report_metrics_db=true', '--access_token='+accessToken, '--test_name='+testName, '--sys_info='+str(sysInfo).strip('[]')]) + except OSError: + print 'Could not execute the test, please check test name' + +if __name__ == "__main__": + main()
\ No newline at end of file diff --git a/test/cpp/qps/user_data.proto b/test/cpp/qps/user_data.proto new file mode 100644 index 0000000000..947ccce60b --- /dev/null +++ b/test/cpp/qps/user_data.proto @@ -0,0 +1,122 @@ +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +import "test/cpp/qps/qpstest.proto"; + +package grpc.testing; + +service UserDataTransfer { + // Sends client info + rpc RecordSingleClientData (SingleUserRecordRequest) returns (SingleUserRecordReply) {} + + rpc RetrieveSingleUserData (SingleUserRetrieveRequest) returns (SingleUserRetrieveReply) {} + + rpc RetrieveAllUsersData (AllUsersRetrieveRequest) returns (AllUsersRetrieveReply) {} +} + +//Metrics to be stored +message Metrics { + optional double qps = 1; + optional double qps_per_core = 2; + optional double perc_lat_50 = 3; + optional double perc_lat_90 = 4; + optional double perc_lat_95 = 5; + optional double perc_lat_99 = 6; + optional double perc_lat_99_point_9 = 7; + optional double server_system_time = 8; + optional double server_user_time = 9; + optional double client_system_time = 10; + optional double client_user_time = 11; +} + +//Timestamped details +message DataDetails { + optional string timestamp = 1; + optional string test_name = 2; + optional string sys_info = 3; + optional Metrics metrics = 4; + optional ClientConfig client_config = 5; + optional ServerConfig server_config = 6; +} + +//User details +message UserDetails { + optional string id = 1; + optional string email = 2; + optional bool verified_email = 3; + optional string name = 4; + optional string given_name = 5; + optional string family_name = 6; + optional string link = 7; + optional string picture = 8; + optional string gender = 9; + optional string locale = 10; + optional string hd = 11; +} + +//Stored to database +message SingleUserDetails { + repeated DataDetails data_details = 1; + optional UserDetails user_details = 2; +} + +//Request for storing a single user's data +message SingleUserRecordRequest { + optional string access_token = 1; + optional string test_name = 2; + optional string sys_info = 3; + optional Metrics metrics = 4; + optional ClientConfig client_config = 5; + optional ServerConfig server_config = 6; +} + +//Reply to request for storing single user's data +message SingleUserRecordReply { +} + +//Request for retrieving single user's data +message SingleUserRetrieveRequest { + optional string user_id = 1; +} + +//Reply for request to retrieve single user's data +message SingleUserRetrieveReply { + optional SingleUserDetails details = 1; +} + +//Request for retrieving all users' data +message AllUsersRetrieveReply { + repeated SingleUserDetails user_data = 1; +} + +//Reply to request for retrieving all users' data +message AllUsersRetrieveRequest { +}
\ No newline at end of file diff --git a/test/cpp/qps/user_data_client.cc b/test/cpp/qps/user_data_client.cc new file mode 100644 index 0000000000..a8ced7559d --- /dev/null +++ b/test/cpp/qps/user_data_client.cc @@ -0,0 +1,114 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "user_data_client.h" + +namespace grpc { +namespace testing { + +//sets the client and server config information +void UserDataClient::setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig) { + clientConfig_ = clientConfig; + serverConfig_ = serverConfig; +} + +//sets the QPS +void UserDataClient::setQPS(double QPS) { + QPS_ = QPS; +} + +//sets the QPS per core +void UserDataClient::setQPSPerCore(double QPSPerCore) { + QPSPerCore_ = QPSPerCore; +} + +//sets the 50th, 90th, 95th, 99th and 99.9th percentile latency +void UserDataClient::setLatencies(double percentileLatency50, double percentileLatency90, + double percentileLatency95, double percentileLatency99, double percentileLatency99Point9) { + percentileLatency50_ = percentileLatency50; + percentileLatency90_ = percentileLatency90; + percentileLatency95_ = percentileLatency95; + percentileLatency99_ = percentileLatency99; + percentileLatency99Point9_ = percentileLatency99Point9; +} + +//sets the server and client, user and system times +void UserDataClient::setTimes(double serverSystemTime, double serverUserTime, + double clientSystemTime, double clientUserTime) { + serverSystemTime_ = serverSystemTime; + serverUserTime_ = serverUserTime; + clientSystemTime_ = clientSystemTime; + clientUserTime_ = clientUserTime; +} + +//sends the data to the performancew database server +int UserDataClient::sendData(std::string access_token, std::string test_name, std::string sys_info) { + //Data record request object + SingleUserRecordRequest singleUserRecordRequest; + + //setting access token, name of the test and the system information + singleUserRecordRequest.set_access_token(access_token); + singleUserRecordRequest.set_test_name(test_name); + singleUserRecordRequest.set_sys_info(sys_info); + + //setting configs + *(singleUserRecordRequest.mutable_client_config()) = clientConfig_; + *(singleUserRecordRequest.mutable_server_config()) = serverConfig_; + + Metrics* metrics = singleUserRecordRequest.mutable_metrics(); + + //setting metrcs in data record request + if(QPS_ != DBL_MIN) metrics->set_qps(QPS_); + if(QPSPerCore_ != DBL_MIN) metrics->set_qps_per_core(QPSPerCore_); + if(percentileLatency50_ != DBL_MIN) metrics->set_perc_lat_50(percentileLatency50_); + if(percentileLatency90_ != DBL_MIN) metrics->set_perc_lat_90(percentileLatency90_); + if(percentileLatency95_ != DBL_MIN) metrics->set_perc_lat_95(percentileLatency95_); + if(percentileLatency99_ != DBL_MIN) metrics->set_perc_lat_99(percentileLatency99_); + if(percentileLatency99Point9_ != DBL_MIN) metrics->set_perc_lat_99_point_9(percentileLatency99Point9_); + if(serverSystemTime_ != DBL_MIN) metrics->set_server_system_time(serverSystemTime_); + if(serverUserTime_ != DBL_MIN) metrics->set_server_user_time(serverUserTime_); + if(clientSystemTime_ != DBL_MIN) metrics->set_client_system_time(clientSystemTime_); + if(clientUserTime_ != DBL_MIN) metrics->set_client_user_time(clientUserTime_); + + SingleUserRecordReply singleUserRecordReply; + ClientContext context; + + Status status = stub_->RecordSingleClientData(&context, singleUserRecordRequest, &singleUserRecordReply); + if (status.IsOk()) { + return 1; //data sent to database successfully + } else { + return -1; //error in data sending + } +} +} //testing +} //grpc
\ No newline at end of file diff --git a/test/cpp/qps/user_data_client.h b/test/cpp/qps/user_data_client.h new file mode 100644 index 0000000000..c2e07ef5cd --- /dev/null +++ b/test/cpp/qps/user_data_client.h @@ -0,0 +1,100 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include <iostream> +#include <memory> +#include <string> +#include <cfloat> + +#include <grpc/grpc.h> +#include <grpc++/channel_arguments.h> +#include <grpc++/channel_interface.h> +#include <grpc++/client_context.h> +#include <grpc++/create_channel.h> +#include <grpc++/credentials.h> +#include <grpc++/status.h> +#include "test/cpp/qps/user_data.grpc.pb.h" + + +namespace grpc{ +namespace testing { + +//Manages data sending to performance database server +class UserDataClient { +public: + UserDataClient(std::shared_ptr<ChannelInterface> channel) + : stub_(UserDataTransfer::NewStub(channel)) {} + + ~UserDataClient() {} + + //sets the client and server config information + void setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig); + + //sets the QPS + void setQPS(double QPS); + + //sets the QPS per core + void setQPSPerCore(double QPSPerCore); + + //sets the 50th, 90th, 95th, 99th and 99.9th percentile latency + void setLatencies(double percentileLatency50, double percentileLatency90, + double percentileLatency95, double percentileLatency99, double percentileLatency99Point9); + + //sets the server and client, user and system times + void setTimes(double serverSystemTime, double serverUserTime, + double clientSystemTime, double clientUserTime); + + //sends the data to the performancew database server + int sendData(std::string access_token, std::string test_name, std::string sys_info); + +private: + std::unique_ptr<UserDataTransfer::Stub> stub_; + ClientConfig clientConfig_; + ServerConfig serverConfig_; + double QPS_ = DBL_MIN; + double QPSPerCore_ = DBL_MIN; + double percentileLatency50_ = DBL_MIN; + double percentileLatency90_ = DBL_MIN; + double percentileLatency95_ = DBL_MIN; + double percentileLatency99_ = DBL_MIN; + double percentileLatency99Point9_ = DBL_MIN; + double serverSystemTime_ = DBL_MIN; + double serverUserTime_ = DBL_MIN; + double clientSystemTime_ = DBL_MIN; + double clientUserTime_ = DBL_MIN; +}; + +} //namespace testing +} //namespace grpc + + diff --git a/test/cpp/util/benchmark_config.cc b/test/cpp/util/benchmark_config.cc index 5b3c1daf5d..250bb7ac8e 100644 --- a/test/cpp/util/benchmark_config.cc +++ b/test/cpp/util/benchmark_config.cc @@ -37,6 +37,14 @@ DEFINE_bool(enable_log_reporter, true, "Enable reporting of benchmark results through GprLog"); +DEFINE_bool(report_metrics_db, false, "True if metrics to be reported to performance database"); + +DEFINE_string(access_token, "", "Authorizing JSON string for leaderboard"); + +DEFINE_string(test_name, "", "Name of the test being executed"); + +DEFINE_string(sys_info, "", "System information"); + // In some distros, gflags is in the namespace google, and in some others, // in gflags. This hack is enabling us to find both. namespace google {} @@ -57,6 +65,11 @@ static std::shared_ptr<Reporter> InitBenchmarkReporters() { composite_reporter->add( std::unique_ptr<Reporter>(new GprLogReporter("LogReporter"))); } + if(FLAGS_report_metrics_db) { + composite_reporter->add( + std::unique_ptr<Reporter>(new UserDatabaseReporter("UserDataReporter", FLAGS_access_token, FLAGS_test_name, FLAGS_sys_info))); + } + return std::shared_ptr<Reporter>(composite_reporter); } |