diff options
Diffstat (limited to 'test/cpp')
| -rw-r--r-- | test/cpp/qps/perf_db.proto | 78 | ||||
| -rw-r--r-- | test/cpp/qps/perf_db_client.cc | 14 | ||||
| -rw-r--r-- | test/cpp/qps/perf_db_client.h | 38 | ||||
| -rw-r--r-- | test/cpp/qps/report.cc | 10 | ||||
| -rw-r--r-- | test/cpp/qps/report.h | 12 | ||||
| -rwxr-xr-x | test/cpp/qps/run_perf_db_test.py | 289 |
6 files changed, 76 insertions, 365 deletions
diff --git a/test/cpp/qps/perf_db.proto b/test/cpp/qps/perf_db.proto index f4f174937c..50070fda5b 100644 --- a/test/cpp/qps/perf_db.proto +++ b/test/cpp/qps/perf_db.proto @@ -27,7 +27,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -syntax = "proto2"; +syntax = "proto3"; import "test/cpp/qps/qpstest.proto"; @@ -44,58 +44,58 @@ service PerfDbTransfer { //Metrics to be stored message Metrics { - optional double qps = 1; - optional double qps_per_core = 2; - optional double perc_lat_50 = 3; - optional double perc_lat_90 = 4; - optional double perc_lat_95 = 5; - optional double perc_lat_99 = 6; - optional double perc_lat_99_point_9 = 7; - optional double server_system_time = 8; - optional double server_user_time = 9; - optional double client_system_time = 10; - optional double client_user_time = 11; + double qps = 1; + double qps_per_core = 2; + double perc_lat_50 = 3; + double perc_lat_90 = 4; + double perc_lat_95 = 5; + double perc_lat_99 = 6; + double perc_lat_99_point_9 = 7; + double server_system_time = 8; + double server_user_time = 9; + double client_system_time = 10; + double client_user_time = 11; } //Timestamped details message DataDetails { - optional string timestamp = 1; - optional string test_name = 2; - optional string sys_info = 3; - optional Metrics metrics = 4; - optional ClientConfig client_config = 5; - optional ServerConfig server_config = 6; + string timestamp = 1; + string test_name = 2; + string sys_info = 3; + Metrics metrics = 4; + ClientConfig client_config = 5; + ServerConfig server_config = 6; } //User details message UserDetails { - optional string id = 1; - optional string email = 2; - optional bool verified_email = 3; - optional string name = 4; - optional string given_name = 5; - optional string family_name = 6; - optional string link = 7; - optional string picture = 8; - optional string gender = 9; - optional string locale = 10; - optional string hd = 11; + string id = 1; + string email = 2; + bool verified_email = 3; + string name = 4; + string given_name = 5; + string family_name = 6; + string link = 7; + string picture = 8; + string gender = 9; + string locale = 10; + string hd = 11; } //Stored to database message SingleUserDetails { repeated DataDetails data_details = 1; - optional UserDetails user_details = 2; + UserDetails user_details = 2; } //Request for storing a single user's data message SingleUserRecordRequest { - optional string access_token = 1; - optional string test_name = 2; - optional string sys_info = 3; - optional Metrics metrics = 4; - optional ClientConfig client_config = 5; - optional ServerConfig server_config = 6; + string access_token = 1; + string test_name = 2; + string sys_info = 3; + Metrics metrics = 4; + ClientConfig client_config = 5; + ServerConfig server_config = 6; } //Reply to request for storing single user's data @@ -104,12 +104,12 @@ message SingleUserRecordReply { //Request for retrieving single user's data message SingleUserRetrieveRequest { - optional string user_id = 1; + string user_id = 1; } //Reply for request to retrieve single user's data message SingleUserRetrieveReply { - optional SingleUserDetails details = 1; + SingleUserDetails details = 1; } //Request for retrieving all users' data @@ -119,4 +119,4 @@ message AllUsersRetrieveReply { //Reply to request for retrieving all users' data message AllUsersRetrieveRequest { -}
\ No newline at end of file +} diff --git a/test/cpp/qps/perf_db_client.cc b/test/cpp/qps/perf_db_client.cc index 6a1fe7e26e..5b92908766 100644 --- a/test/cpp/qps/perf_db_client.cc +++ b/test/cpp/qps/perf_db_client.cc @@ -31,30 +31,30 @@ * */ -#include "perf_db_client.h" +#include "test/cpp/qps/perf_db_client.h" namespace grpc { namespace testing { //sets the client and server config information -void PerfDbClient::setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig) const { +void PerfDbClient::setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig) { clientConfig_ = clientConfig; serverConfig_ = serverConfig; } //sets the QPS -void PerfDbClient::setQPS(double QPS) const { +void PerfDbClient::setQPS(double QPS) { QPS_ = QPS; } //sets the QPS per core -void PerfDbClient::setQPSPerCore(double QPSPerCore) const { +void PerfDbClient::setQPSPerCore(double QPSPerCore) { QPSPerCore_ = QPSPerCore; } //sets the 50th, 90th, 95th, 99th and 99.9th percentile latency void PerfDbClient::setLatencies(double percentileLatency50, double percentileLatency90, - double percentileLatency95, double percentileLatency99, double percentileLatency99Point9) const { + double percentileLatency95, double percentileLatency99, double percentileLatency99Point9) { percentileLatency50_ = percentileLatency50; percentileLatency90_ = percentileLatency90; percentileLatency95_ = percentileLatency95; @@ -64,7 +64,7 @@ void PerfDbClient::setLatencies(double percentileLatency50, double percentileLat //sets the server and client, user and system times void PerfDbClient::setTimes(double serverSystemTime, double serverUserTime, - double clientSystemTime, double clientUserTime) const { + double clientSystemTime, double clientUserTime) { serverSystemTime_ = serverSystemTime; serverUserTime_ = serverUserTime; clientSystemTime_ = clientSystemTime; @@ -72,7 +72,7 @@ void PerfDbClient::setTimes(double serverSystemTime, double serverUserTime, } //sends the data to the performancew database server -int PerfDbClient::sendData(std::string access_token, std::string test_name, std::string sys_info) const { +int PerfDbClient::sendData(std::string access_token, std::string test_name, std::string sys_info) { //Data record request object SingleUserRecordRequest singleUserRecordRequest; diff --git a/test/cpp/qps/perf_db_client.h b/test/cpp/qps/perf_db_client.h index 4a63dbf3d2..058cd6a207 100644 --- a/test/cpp/qps/perf_db_client.h +++ b/test/cpp/qps/perf_db_client.h @@ -59,40 +59,40 @@ public: ~PerfDbClient() {} //sets the client and server config information - void setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig) const; + void setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig); //sets the QPS - void setQPS(double QPS) const; + void setQPS(double QPS); //sets the QPS per core - void setQPSPerCore(double QPSPerCore) const; + void setQPSPerCore(double QPSPerCore); //sets the 50th, 90th, 95th, 99th and 99.9th percentile latency void setLatencies(double percentileLatency50, double percentileLatency90, - double percentileLatency95, double percentileLatency99, double percentileLatency99Point9) const; + double percentileLatency95, double percentileLatency99, double percentileLatency99Point9); //sets the server and client, user and system times void setTimes(double serverSystemTime, double serverUserTime, - double clientSystemTime, double clientUserTime) const; + double clientSystemTime, double clientUserTime); //sends the data to the performancew database server - int sendData(std::string access_token, std::string test_name, std::string sys_info) const; + int sendData(std::string access_token, std::string test_name, std::string sys_info); private: std::unique_ptr<PerfDbTransfer::Stub> stub_; - mutable ClientConfig clientConfig_; - mutable ServerConfig serverConfig_; - mutable double QPS_ = DBL_MIN; - mutable double QPSPerCore_ = DBL_MIN; - mutable double percentileLatency50_ = DBL_MIN; - mutable double percentileLatency90_ = DBL_MIN; - mutable double percentileLatency95_ = DBL_MIN; - mutable double percentileLatency99_ = DBL_MIN; - mutable double percentileLatency99Point9_ = DBL_MIN; - mutable double serverSystemTime_ = DBL_MIN; - mutable double serverUserTime_ = DBL_MIN; - mutable double clientSystemTime_ = DBL_MIN; - mutable double clientUserTime_ = DBL_MIN; + ClientConfig clientConfig_; + ServerConfig serverConfig_; + double QPS_ = DBL_MIN; + double QPSPerCore_ = DBL_MIN; + double percentileLatency50_ = DBL_MIN; + double percentileLatency90_ = DBL_MIN; + double percentileLatency95_ = DBL_MIN; + double percentileLatency99_ = DBL_MIN; + double percentileLatency99Point9_ = DBL_MIN; + double serverSystemTime_ = DBL_MIN; + double serverUserTime_ = DBL_MIN; + double clientSystemTime_ = DBL_MIN; + double clientUserTime_ = DBL_MIN; }; } //namespace testing diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index 47d602a4c2..59c0da31fb 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -118,7 +118,7 @@ void GprLogReporter::ReportTimes(const ScenarioResult& result) { [](ResourceUsage u) { return u.wall_time; })); } -void PerfDbReporter::ReportQPS(const ScenarioResult& result) const { +void PerfDbReporter::ReportQPS(const ScenarioResult& result) { auto qps = result.latencies.Count() / average(result.client_resources, [](ResourceUsage u) { return u.wall_time; }); @@ -127,7 +127,7 @@ void PerfDbReporter::ReportQPS(const ScenarioResult& result) const { perfDbClient_.setConfigs(result.client_config, result.server_config); } -void PerfDbReporter::ReportQPSPerCore(const ScenarioResult& result) const { +void PerfDbReporter::ReportQPSPerCore(const ScenarioResult& result) { auto qps = result.latencies.Count() / average(result.client_resources, [](ResourceUsage u) { return u.wall_time; }); @@ -139,7 +139,7 @@ void PerfDbReporter::ReportQPSPerCore(const ScenarioResult& result) const { perfDbClient_.setConfigs(result.client_config, result.server_config); } -void PerfDbReporter::ReportLatency(const ScenarioResult& result) const { +void PerfDbReporter::ReportLatency(const ScenarioResult& result) { perfDbClient_.setLatencies(result.latencies.Percentile(50) / 1000, result.latencies.Percentile(90) / 1000, result.latencies.Percentile(95) / 1000, @@ -148,7 +148,7 @@ void PerfDbReporter::ReportLatency(const ScenarioResult& result) const { perfDbClient_.setConfigs(result.client_config, result.server_config); } -void PerfDbReporter::ReportTimes(const ScenarioResult& result) const { +void PerfDbReporter::ReportTimes(const ScenarioResult& result) { double serverSystemTime = 100.0 * sum(result.server_resources, [](ResourceUsage u) { return u.system_time; }) / sum(result.server_resources, @@ -171,7 +171,7 @@ void PerfDbReporter::ReportTimes(const ScenarioResult& result) const { perfDbClient_.setConfigs(result.client_config, result.server_config); } -void PerfDbReporter::SendData() const { +void PerfDbReporter::SendData() { //send data to performance database int dataState = perfDbClient_.sendData(access_token_, test_name_, sys_info_); diff --git a/test/cpp/qps/report.h b/test/cpp/qps/report.h index 2ec7fe73f0..bba26b990c 100644 --- a/test/cpp/qps/report.h +++ b/test/cpp/qps/report.h @@ -109,7 +109,7 @@ class PerfDbReporter : public Reporter { public: PerfDbReporter(const string& name, const string& access_token, const string& test_name, const string& sys_info, const string& server_address) : Reporter(name), access_token_(access_token), test_name_(test_name), sys_info_(sys_info) { - perfDbClient.init(grpc::CreateChannel(server_address, grpc::InsecureCredentials(), ChannelArguments())); + perfDbClient_.init(grpc::CreateChannel(server_address, grpc::InsecureCredentials(), ChannelArguments())); } ~PerfDbReporter() { SendData(); }; @@ -118,11 +118,11 @@ class PerfDbReporter : public Reporter { std::string access_token_; std::string test_name_; std::string sys_info_; - void ReportQPS(const ScenarioResult& result) const GRPC_OVERRIDE; - void ReportQPSPerCore(const ScenarioResult& result) const GRPC_OVERRIDE; - void ReportLatency(const ScenarioResult& result) const GRPC_OVERRIDE; - void ReportTimes(const ScenarioResult& result) const GRPC_OVERRIDE; - void SendData() const; + void ReportQPS(const ScenarioResult& result) GRPC_OVERRIDE; + void ReportQPSPerCore(const ScenarioResult& result) GRPC_OVERRIDE; + void ReportLatency(const ScenarioResult& result) GRPC_OVERRIDE; + void ReportTimes(const ScenarioResult& result) GRPC_OVERRIDE; + void SendData(); }; } // namespace testing diff --git a/test/cpp/qps/run_perf_db_test.py b/test/cpp/qps/run_perf_db_test.py deleted file mode 100755 index a8e69ea1f0..0000000000 --- a/test/cpp/qps/run_perf_db_test.py +++ /dev/null @@ -1,289 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2015, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# - -import os -import sys -import re -import urllib2 -import urllib -import json -import time -import subprocess -import fnmatch -import argparse - -CLIENT_ID = '1018396037782-tv81fshn76nemr24uuhuginceb9hni2m.apps.googleusercontent.com' -CLIENT_SECRET = '_HGHXg4DAA59r4w4x8p6ARzD' -GRANT_TYPE = 'http://oauth.net/grant_type/device/1.0' -AUTH_TOKEN_LINK = 'https://www.googleapis.com/oauth2/v3/token' -GOOGLE_ACCOUNTS_LINK = 'https://accounts.google.com/o/oauth2/device/code' -USER_INFO_LINK = 'https://www.googleapis.com/oauth2/v1/userinfo' - -parser = argparse.ArgumentParser(description='Report metrics to performance database') -parser.add_argument('--test', type=str, help='Name of the test to be executed') -parser.add_argument('--email', type=str, help='Gmail address of the user') -parser.add_argument('--server_address', type=str, default='localhost:50052', help='Address of the performance database server') -parser.add_argument('--tokens_dir', type=str, default=os.path.expanduser('~')+'/.grpc/access_tokens', help='Path to the access tokens directory') - -# Fetches JSON reply object, given a url and parameters -def fetchJSON(url, paramDict): - if len(paramDict) == 0: - req = urllib2.Request(url) - else: - data = urllib.urlencode(paramDict) - req = urllib2.Request(url, data) - - try: - response = urllib2.urlopen(req) - result = response.read() - - except urllib2.HTTPError, error: - result = error.read() - - return result - -# Fetch user info; used to check if access token is valid -def getUserInfo(accessToken): - url = USER_INFO_LINK + '?access_token=' + accessToken - paramDict = {} - JSONBody = fetchJSON(url, paramDict) - data = json.loads(JSONBody) - - return data - -# Returns true if stored access token is valid -def isAccessTokenValid(accessToken): - data = getUserInfo(accessToken); - - if 'id' in data: - return True - else: - return False - -# Returns user id given a working access token -def getUserId(accessToken): - data = getUserInfo(accessToken) - - email = data['email'] - userId = getUserIdFromEmail(email) - - return userId - -# Extracts a unique user id from an email address -def getUserIdFromEmail(email): - email = email.split('@')[0].lower() # take username and convert to lower case - userId = re.sub('[.]', '', email) # remove periods - - return userId - -# Use an existing access token -def useAccessToken(userTokFile): - with open(userTokFile, "r") as data_file: - data = json.load(data_file) # load JSON data from file - accessToken = data["access_token"] - - # If access token has gone stale, refresh it - if not isAccessTokenValid(accessToken): - return refreshAccessToken(data["refresh_token"], userTokFile) - - return accessToken - -# refresh stale access token -def refreshAccessToken(refreshToken, userTokFile): - # Parameters for request - paramDict = {'refresh_token':refreshToken, 'client_id':CLIENT_ID, 'client_secret':CLIENT_SECRET, 'grant_type':'refresh_token'} - # Fetch reply to request - JSONBody = fetchJSON(AUTH_TOKEN_LINK, paramDict) - data = json.loads(JSONBody) - - if not 'access_token' in data: - # Refresh token has gone stale, re-authentication required - return reauthenticate() - else: - # write fresh access token to tokens file - tokenData = {} - - with open(userTokFile, "r") as data_file: - tokenData = json.load(data_file) - - with open(userTokFile, "w") as data_file: - tokenData['access_token'] = data['access_token'] - json.dump(tokenData, data_file) - - # return fresh access token - return data['access_token'] - -def reauthenticate(tokensDir): - # Create folder if not created already - if not os.path.exists(tokensDir): - os.makedirs(tokensDir) - os.chmod(tokensDir, 0700) - - # Request parameters - paramDict = {'client_id':CLIENT_ID, 'scope':'email profile'} - JSONBody = fetchJSON(GOOGLE_ACCOUNTS_LINK, paramDict) - data = json.loads(JSONBody) - - print 'User authorization required\n' - print 'Please use the following code in you browser: ', data['user_code'] # Code to be entered by user in browser - print 'Verification URL: ', data['verification_url'] # Authentication link - print '\nAwaiting user authorization. May take a few more seconds after authorizing...\n' - - authData = {} - - while not 'access_token' in authData: - # Request parameters - authDict = {'client_id':CLIENT_ID, 'client_secret':CLIENT_SECRET, 'code':data['device_code'], 'grant_type':GRANT_TYPE} - JSONBody = fetchJSON(AUTH_TOKEN_LINK, authDict) - authData = json.loads(JSONBody) - # If server pinged too quickly, will get slowdown message; need to wait for specified interval - time.sleep(data['interval']) - - # File to write tokens - newUserTokFile = tokensDir + '/' + getUserId(authData['access_token']) - - # Write tokens to file - with open(newUserTokFile, "w") as data_file: - os.chmod(newUserTokFile, 0600) - json.dump(authData, data_file) - - # return working access token - return authData['access_token'] - -# Fetch a working access token given user entered email id; authntication may be required -def getAccessToken(email, tokensDir): - # Get unique user id from email address - userId = getUserIdFromEmail(email) - - # Token file - userTokFile = tokensDir + '/' + userId - - accessToken = '' - - if os.path.exists(userTokFile): - # File containing access token exists; unless refresh token has expired, user authentication will not be required - accessToken = useAccessToken(userTokFile) - else: - # User authentication required - accessToken = reauthenticate(tokensDir) - - return accessToken - -# If user has not entered full path to test, recursively searches for given test in parent folders -def findTestPath(test): - # If user entered full path to test, return it - if(os.path.isfile(test)): - return test - - testName = test.split('/')[-1] # Extract just test name - testPath = '' - - # Search for test - for root, dirnames, filenames in os.walk('../../../'): - for fileName in fnmatch.filter(filenames, testName): - testPath = os.path.join(root, fileName) - - return testPath - -def getSysInfo(): - # Fetch system information - sysInfo = os.popen('lscpu').readlines() - - NICs = os.popen('ifconfig | cut -c1-8 | sed \'/^\s*$/d\' | sort -u').readlines() - nicAddrs = os.popen('ifconfig | grep -oE "inet addr:([0-9]{1,3}\.){3}[0-9]{1,3}"').readlines() - - nicInfo = [] - - for i in range(0, len(NICs)): - NIC = NICs[i] - NIC = re.sub(r'[^\w]', '', NIC) - - ethtoolProcess = subprocess.Popen(["ethtool",NIC], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - ethtoolResult = ethtoolProcess.communicate()[0] - - ethtoolResultList = ethtoolResult.split('\n\t') - for ethtoolString in ethtoolResultList: - if ethtoolString.startswith('Speed'): - ethtoolString = ethtoolString.split(':')[1] - ethtoolString = ethtoolString.replace('Mb/s',' Mbps') - nicInfo.append('NIC ' + NIC + ' speed: ' + ethtoolString + '\n') - nicInfo.append(NIC + ' inet address: ' + nicAddrs[i].split(':')[1]) - - print 'Obtaining network info....' - tcp_rr_rate = str(os.popen('netperf -t TCP_RR -v 0').readlines()[1]) - print 'Network info obtained' - - nicInfo.append('TCP RR transmission rate per sec: ' + tcp_rr_rate + '\n') - sysInfo = sysInfo + nicInfo - - return sysInfo - -def main(argv): - args = parser.parse_args() - - tokensDir = args.tokens_dir - - try: - # Fetch working access token - accessToken = getAccessToken(args.email, tokensDir) - except AttributeError: - print '\nError: Please provide email address as an argument\n' - sys.exit(1) - except Exception, e: - print e, ' \nError in authentication\n' - sys.exit(1) - - # Address of the performance database server - serverAddress = args.server_address - - # Get path to test - try: - testPath = findTestPath(args.test) - except TypeError: - print '\nError: Please provide test name/path as argument\n' - sys.exit(1) - - # Get name of the test - testName = testPath.split('/')[-1] - - # Get the system information - sysInfo = getSysInfo() - - try: - print '\nBeginning test:\n' - # Run the test - subprocess.call([testPath, '--report_metrics_db=true', '--access_token='+accessToken, '--test_name='+testName, '--sys_info='+str(sysInfo).strip('[]'), '--server_address='+serverAddress]) - except OSError: - print 'Could not execute the test' - -if __name__ == "__main__": - main(sys.argv)
\ No newline at end of file |