Merge pull request #6084 from sreecha/nodejs_stress

Add nodejs stress tests to the test-framework

7 files changed, 408 insertions, 40 deletions

diff --git a/tools/run_tests/stress_test/configs/asan.json b/tools/run_tests/stress_test/configs/asan.json
new file mode 100644
index 0000000000..768088d93d
--- /dev/null
+++ b/tools/run_tests/stress_test/configs/asan.json
@@ -0,0 +1,86 @@
+{
+  "dockerImages": {
+    "grpc_stress_cxx_asan" : {
+      "buildScript": "tools/jenkins/build_interop_stress_image.sh",
+      "dockerFileDir": "grpc_interop_stress_cxx",
+      "buildType": "asan"
+    }
+  },
+
+  "clientTemplates": {
+    "baseTemplates": {
+      "default": {
+        "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py",
+        "pollIntervalSecs": 60,
+        "clientArgs": {
+          "num_channels_per_server":5,
+          "num_stubs_per_channel":10,
+          "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1",
+          "metrics_port": 8081,
+          "metrics_collection_interval_secs":60
+        },
+        "metricsPort": 8081,
+        "metricsArgs": {
+          "metrics_server_address": "localhost:8081",
+          "total_only": "true"
+        }
+      }
+    },
+    "templates": {
+      "cxx_client_asan": {
+        "baseTemplate": "default",
+        "stressClientCmd": ["/var/local/git/grpc/bins/asan/stress_test"],
+        "metricsClientCmd": ["/var/local/git/grpc/bins/asan/metrics_client"]
+      }
+    }
+  },
+
+  "serverTemplates": {
+    "baseTemplates":{
+      "default": {
+        "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py",
+        "serverPort": 8080,
+        "serverArgs": {
+          "port": 8080
+        }
+      }
+    },
+    "templates": {
+      "cxx_server_asan": {
+        "baseTemplate": "default",
+        "stressServerCmd": ["/var/local/git/grpc/bins/asan/interop_server"]
+      }
+    }
+  },
+
+  "testMatrix": {
+    "serverPodSpecs": {
+      "stress-server-asan": {
+        "serverTemplate": "cxx_server_asan",
+        "dockerImage": "grpc_stress_cxx_asan",
+        "numInstances": 1
+      }
+    },
+
+    "clientPodSpecs": {
+      "stress-client-asan": {
+        "clientTemplate": "cxx_client_asan",
+        "dockerImage": "grpc_stress_cxx_asan",
+        "numInstances": 20,
+        "serverPodSpec": "stress-server-asan"
+      }
+    }
+  },
+
+  "globalSettings": {
+    "buildDockerImages": true,
+    "pollIntervalSecs": 60,
+    "testDurationSecs": 7200,
+    "kubernetesProxyPort": 8003,
+    "datasetIdNamePrefix": "stress_test_asan",
+    "summaryTableId": "summary",
+    "qpsTableId": "qps",
+    "podWarmupSecs": 60
+  }
+}
+
diff --git a/tools/run_tests/stress_test/configs/node-cxx.json b/tools/run_tests/stress_test/configs/node-cxx.json
new file mode 100644
index 0000000000..c4245bf9df
--- /dev/null
+++ b/tools/run_tests/stress_test/configs/node-cxx.json
@@ -0,0 +1,97 @@
+{
+  "dockerImages": {
+    "grpc_stress_cxx_opt" : {
+      "buildScript": "tools/jenkins/build_interop_stress_image.sh",
+      "dockerFileDir": "grpc_interop_stress_cxx",
+      "buildType": "opt"
+    },
+   "grpc_stress_node": {
+     "buildScript": "tools/jenkins/build_interop_stress_image.sh",
+     "dockerFileDir": "grpc_interop_stress_node"
+   }
+  },
+
+  "clientTemplates": {
+    "baseTemplates": {
+      "default": {
+        "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py",
+        "pollIntervalSecs": 60,
+        "clientArgs": {
+          "num_channels_per_server":5,
+          "num_stubs_per_channel":10,
+          "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1",
+          "metrics_port": 8081
+        },
+        "metricsPort": 8081,
+        "metricsArgs": {
+          "metrics_server_address": "localhost:8081",
+          "total_only": "true"
+        }
+      }
+    },
+    "templates": {
+      "node_client": {
+        "baseTemplate": "default",
+        "stressClientCmd": [
+          "/var/local/git/grpc/tools/gcp/stress_test/run_node.sh",
+          "node",
+          "/var/local/git/grpc/src/node/stress/stress_client.js"
+        ],
+        "metricsClientCmd": [
+          "/var/local/git/grpc/tools/gcp/stress_test/run_node.sh",
+          "node",
+          "/var/local/git/grpc/src/node/stress/metrics_client.js"
+        ]
+      }
+    }
+  },
+
+  "serverTemplates": {
+    "baseTemplates":{
+      "default": {
+        "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py",
+        "serverPort": 8080,
+        "serverArgs": {
+          "port": 8080
+        }
+      }
+    },
+    "templates": {
+      "cxx_server_opt": {
+        "baseTemplate": "default",
+        "stressServerCmd": ["/var/local/git/grpc/bins/opt/interop_server"]
+      }
+    }
+  },
+
+  "testMatrix": {
+    "serverPodSpecs": {
+      "stress-server-cxx-opt": {
+        "serverTemplate": "cxx_server_opt",
+        "dockerImage": "grpc_stress_cxx_opt",
+        "numInstances": 1
+      }
+    },
+
+    "clientPodSpecs": {
+      "stress-client-node": {
+        "clientTemplate": "node_client",
+        "dockerImage": "grpc_stress_node",
+        "numInstances": 20,
+        "serverPodSpec": "stress-server-cxx-opt"
+      }
+    }
+  },
+
+  "globalSettings": {
+    "buildDockerImages": true,
+    "pollIntervalSecs": 60,
+    "testDurationSecs": 7200,
+    "kubernetesProxyPort": 8006,
+    "datasetIdNamePrefix": "stress_test_node_cxx_opt",
+    "summaryTableId": "summary",
+    "qpsTableId": "qps",
+    "podWarmupSecs": 60
+  }
+}
+
diff --git a/tools/run_tests/stress_test/configs/node.json b/tools/run_tests/stress_test/configs/node.json
new file mode 100644
index 0000000000..7a48c56a5e
--- /dev/null
+++ b/tools/run_tests/stress_test/configs/node.json
@@ -0,0 +1,96 @@
+{
+  "dockerImages": {
+    "grpc_stress_node" : {
+      "buildScript": "tools/jenkins/build_interop_stress_image.sh",
+      "dockerFileDir": "grpc_interop_stress_node"
+    }
+  },
+
+  "clientTemplates": {
+    "baseTemplates": {
+      "default": {
+        "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py",
+        "pollIntervalSecs": 60,
+        "clientArgs": {
+          "num_channels_per_server":5,
+          "num_stubs_per_channel":10,
+          "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1",
+          "metrics_port": 8081
+        },
+        "metricsPort": 8081,
+        "metricsArgs": {
+          "metrics_server_address": "localhost:8081",
+          "total_only": "true"
+        }
+      }
+    },
+    "templates": {
+      "node_client": {
+        "baseTemplate": "default",
+        "stressClientCmd": [
+          "/var/local/git/grpc/tools/gcp/stress_test/run_node.sh",
+          "node",
+          "/var/local/git/grpc/src/node/stress/stress_client.js"
+        ],
+        "metricsClientCmd": [
+          "/var/local/git/grpc/tools/gcp/stress_test/run_node.sh",
+          "node",
+          "/var/local/git/grpc/src/node/stress/metrics_client.js"
+        ]
+      }
+    }
+  },
+
+  "serverTemplates": {
+    "baseTemplates":{
+      "default": {
+        "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py",
+        "serverPort": 8080,
+        "serverArgs": {
+          "port": 8080
+        }
+      }
+    },
+    "templates": {
+      "node_server": {
+        "baseTemplate": "default",
+        "stressServerCmd": [
+          "/var/local/git/grpc/tools/gcp/stress_test/run_node.sh",
+          "node",
+          "/var/local/git/grpc/src/node/interop/interop_server.js"
+        ]
+      }
+    }
+  },
+
+  "testMatrix": {
+    "serverPodSpecs": {
+      "node-stress-server": {
+        "serverTemplate": "node_server",
+        "dockerImage": "grpc_stress_node",
+        "numInstances": 1
+      }
+    },
+
+    "clientPodSpecs": {
+      "node-stress-client": {
+        "clientTemplate": "node_client",
+        "dockerImage": "grpc_stress_node",
+        "numInstances": 15,
+        "serverPodSpec": "node-stress-server"
+      }
+    }
+  },
+
+  "globalSettings": {
+    "buildDockerImages": true,
+    "pollIntervalSecs": 60,
+    "testDurationSecs": 7200,
+    "kubernetesProxyPort": 8005,
+    "datasetIdNamePrefix": "stress_test_node",
+    "summaryTableId": "summary",
+    "qpsTableId": "qps",
+    "podWarmupSecs": 60
+  }
+}
+
diff --git a/tools/run_tests/stress_test/configs/opt-tsan-asan.json b/tools/run_tests/stress_test/configs/opt-tsan-asan.json
index 1dc2d3fe08..4f172ef30b 100644
--- a/tools/run_tests/stress_test/configs/opt-tsan-asan.json
+++ b/tools/run_tests/stress_test/configs/opt-tsan-asan.json
@@ -27,7 +27,7 @@
           "num_stubs_per_channel":10,
           "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1",
           "metrics_port": 8081,
-          "metrics_collection_interval_secs":60
+          "metrics_collection_interval_secs": 60
         },
         "metricsPort": 8081,
         "metricsArgs": {
@@ -39,18 +39,18 @@
     "templates": {
       "cxx_client_opt": {
         "baseTemplate": "default",
-        "clientImagePath": "/var/local/git/grpc/bins/opt/stress_test",
-        "metricsClientImagePath": "/var/local/git/grpc/bins/opt/metrics_client"
+        "stressClientCmd": ["/var/local/git/grpc/bins/opt/stress_test"],
+        "metricsClientCmd": ["/var/local/git/grpc/bins/opt/metrics_client"]
       },
       "cxx_client_tsan": {
         "baseTemplate": "default",
-        "clientImagePath": "/var/local/git/grpc/bins/tsan/stress_test",
-        "metricsClientImagePath": "/var/local/git/grpc/bins/tsan/metrics_client"
+        "stressClientCmd": ["/var/local/git/grpc/bins/tsan/stress_test"],
+        "metricsClientCmd": ["/var/local/git/grpc/bins/tsan/metrics_client"]
       },
     "cxx_client_asan": {
         "baseTemplate": "default",
-        "clientImagePath": "/var/local/git/grpc/bins/asan/stress_test",
-        "metricsClientImagePath": "/var/local/git/grpc/bins/asan/metrics_client"
+        "stressClientCmd": ["/var/local/git/grpc/bins/asan/stress_test"],
+        "metricsClientCmd": ["/var/local/git/grpc/bins/asan/metrics_client"]
       }
     }
   },
@@ -68,15 +68,15 @@
     "templates": {
       "cxx_server_opt": {
         "baseTemplate": "default",
-        "serverImagePath": "/var/local/git/grpc/bins/opt/interop_server"
+        "stressServerCmd": ["/var/local/git/grpc/bins/opt/interop_server"]
       },
       "cxx_server_tsan": {
         "baseTemplate": "default",
-        "serverImagePath": "/var/local/git/grpc/bins/tsan/interop_server"
+        "stressServerCmd": ["/var/local/git/grpc/bins/tsan/interop_server"]
       },
     "cxx_server_asan": {
         "baseTemplate": "default",
-        "serverImagePath": "/var/local/git/grpc/bins/asan/interop_server"
+        "stressServerCmd": ["/var/local/git/grpc/bins/asan/interop_server"]
       }
     }
   },
@@ -104,19 +104,19 @@
       "stress-client-opt": {
         "clientTemplate": "cxx_client_opt",
         "dockerImage": "grpc_stress_cxx_opt",
-        "numInstances": 3,
+        "numInstances": 5,
         "serverPodSpec": "stress-server-opt"
       },
       "stress-client-tsan": {
         "clientTemplate": "cxx_client_tsan",
         "dockerImage": "grpc_stress_cxx_tsan",
-        "numInstances": 3,
+        "numInstances": 10,
         "serverPodSpec": "stress-server-tsan"
       },
       "stress-client-asan": {
         "clientTemplate": "cxx_client_asan",
         "dockerImage": "grpc_stress_cxx_asan",
-        "numInstances": 3,
+        "numInstances": 10,
         "serverPodSpec": "stress-server-asan"
       }
     }
@@ -126,7 +126,7 @@
     "buildDockerImages": true,
     "pollIntervalSecs": 60,
     "testDurationSecs": 7200,
-    "kubernetesProxyPort": 8001,
+    "kubernetesProxyPort": 8004,
     "datasetIdNamePrefix": "stress_test_opt_tsan",
     "summaryTableId": "summary",
     "qpsTableId": "qps",
diff --git a/tools/run_tests/stress_test/configs/opt.json b/tools/run_tests/stress_test/configs/opt.json
index 7fc024034b..ffd4a704c3 100644
--- a/tools/run_tests/stress_test/configs/opt.json
+++ b/tools/run_tests/stress_test/configs/opt.json
@@ -17,7 +17,7 @@
           "num_stubs_per_channel":10,
           "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1",
           "metrics_port": 8081,
-          "metrics_collection_interval_secs":60
+          "metrics_collection_interval_secs": 60
         },
         "metricsPort": 8081,
         "metricsArgs": {
@@ -29,8 +29,8 @@
     "templates": {
       "cxx_client_opt": {
         "baseTemplate": "default",
-        "clientImagePath": "/var/local/git/grpc/bins/opt/stress_test",
-        "metricsClientImagePath": "/var/local/git/grpc/bins/opt/metrics_client"
+        "stressClientCmd": ["/var/local/git/grpc/bins/opt/stress_test"],
+        "metricsClientCmd": ["/var/local/git/grpc/bins/opt/metrics_client"]
       }
     }
   },
@@ -48,7 +48,7 @@
     "templates": {
       "cxx_server_opt": {
         "baseTemplate": "default",
-        "serverImagePath": "/var/local/git/grpc/bins/opt/interop_server"
+        "stressServerCmd": ["/var/local/git/grpc/bins/opt/interop_server"]
       }
     }
   },
@@ -74,8 +74,8 @@
 
   "globalSettings": {
     "buildDockerImages": true,
-    "pollIntervalSecs": 10,
-    "testDurationSecs": 120,
+    "pollIntervalSecs": 60,
+    "testDurationSecs": 7200,
     "kubernetesProxyPort": 8001,
     "datasetIdNamePrefix": "stress_test_opt",
     "summaryTableId": "summary",
diff --git a/tools/run_tests/stress_test/configs/tsan.json b/tools/run_tests/stress_test/configs/tsan.json
new file mode 100644
index 0000000000..f8d3f878e1
--- /dev/null
+++ b/tools/run_tests/stress_test/configs/tsan.json
@@ -0,0 +1,86 @@
+{
+  "dockerImages": {
+    "grpc_stress_cxx_tsan" : {
+      "buildScript": "tools/jenkins/build_interop_stress_image.sh",
+      "dockerFileDir": "grpc_interop_stress_cxx",
+      "buildType": "tsan"
+    }
+  },
+
+  "clientTemplates": {
+    "baseTemplates": {
+      "default": {
+        "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py",
+        "pollIntervalSecs": 60,
+        "clientArgs": {
+          "num_channels_per_server":5,
+          "num_stubs_per_channel":10,
+          "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1",
+          "metrics_port": 8081,
+          "metrics_collection_interval_secs":60
+        },
+        "metricsPort": 8081,
+        "metricsArgs": {
+          "metrics_server_address": "localhost:8081",
+          "total_only": "true"
+        }
+      }
+    },
+    "templates": {
+      "cxx_client_tsan": {
+        "baseTemplate": "default",
+        "stressClientCmd": ["/var/local/git/grpc/bins/tsan/stress_test"],
+        "metricsClientCmd": ["/var/local/git/grpc/bins/tsan/metrics_client"]
+      }
+    }
+  },
+
+  "serverTemplates": {
+    "baseTemplates":{
+      "default": {
+        "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py",
+        "serverPort": 8080,
+        "serverArgs": {
+          "port": 8080
+        }
+      }
+    },
+    "templates": {
+      "cxx_server_tsan": {
+        "baseTemplate": "default",
+        "stressServerCmd": ["/var/local/git/grpc/bins/tsan/interop_server"]
+      }
+    }
+  },
+
+  "testMatrix": {
+    "serverPodSpecs": {
+      "stress-server-tsan": {
+        "serverTemplate": "cxx_server_tsan",
+        "dockerImage": "grpc_stress_cxx_tsan",
+        "numInstances": 1
+      }
+    },
+
+    "clientPodSpecs": {
+      "stress-client-tsan": {
+        "clientTemplate": "cxx_client_tsan",
+        "dockerImage": "grpc_stress_cxx_tsan",
+        "numInstances": 20,
+        "serverPodSpec": "stress-server-tsan"
+      }
+    }
+  },
+
+  "globalSettings": {
+    "buildDockerImages": true,
+    "pollIntervalSecs": 60,
+    "testDurationSecs": 7200,
+    "kubernetesProxyPort": 8002,
+    "datasetIdNamePrefix": "stress_test_tsan",
+    "summaryTableId": "summary",
+    "qpsTableId": "qps",
+    "podWarmupSecs": 60
+  }
+}
+
diff --git a/tools/run_tests/stress_test/run_on_gke.py b/tools/run_tests/stress_test/run_on_gke.py
index 9d4ac7ac58..db3ba28346 100755
--- a/tools/run_tests/stress_test/run_on_gke.py
+++ b/tools/run_tests/stress_test/run_on_gke.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python2.7
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -67,12 +67,12 @@ class GlobalSettings:
 class ClientTemplate:
   """ Contains all the common settings that are used by a stress client """
 
-  def __init__(self, name, client_image_path, metrics_client_image_path,
-               metrics_port, wrapper_script_path, poll_interval_secs,
-               client_args_dict, metrics_args_dict):
+  def __init__(self, name, stress_client_cmd, metrics_client_cmd, metrics_port,
+               wrapper_script_path, poll_interval_secs, client_args_dict,
+               metrics_args_dict):
     self.name = name
-    self.client_image_path = client_image_path
-    self.metrics_client_image_path = metrics_client_image_path
+    self.stress_client_cmd = stress_client_cmd
+    self.metrics_client_cmd = metrics_client_cmd
     self.metrics_port = metrics_port
     self.wrapper_script_path = wrapper_script_path
     self.poll_interval_secs = poll_interval_secs
@@ -83,10 +83,10 @@ class ClientTemplate:
 class ServerTemplate:
   """ Contains all the common settings used by a stress server """
 
-  def __init__(self, name, server_image_path, wrapper_script_path, server_port,
+  def __init__(self, name, server_cmd, wrapper_script_path, server_port,
                server_args_dict):
     self.name = name
-    self.server_image_path = server_image_path
+    self.server_cmd = server_cmd
     self.wrapper_script_path = wrapper_script_path
     self.server_port = server_port
     self.server_args_dict = server_args_dict
@@ -240,7 +240,7 @@ class Gke:
     server_env = self.gke_env.copy()
     server_env.update({
         'STRESS_TEST_IMAGE_TYPE': 'SERVER',
-        'STRESS_TEST_IMAGE': server_pod_spec.template.server_image_path,
+        'STRESS_TEST_CMD': server_pod_spec.template.server_cmd,
         'STRESS_TEST_ARGS_STR': self._args_dict_to_str(
             server_pod_spec.template.server_args_dict)
     })
@@ -282,11 +282,10 @@ class Gke:
     client_env = self.gke_env.copy()
     client_env.update({
         'STRESS_TEST_IMAGE_TYPE': 'CLIENT',
-        'STRESS_TEST_IMAGE': client_pod_spec.template.client_image_path,
+        'STRESS_TEST_CMD': client_pod_spec.template.stress_client_cmd,
         'STRESS_TEST_ARGS_STR': self._args_dict_to_str(
             client_pod_spec.get_client_args_dict()),
-        'METRICS_CLIENT_IMAGE':
-            client_pod_spec.template.metrics_client_image_path,
+        'METRICS_CLIENT_CMD': client_pod_spec.template.metrics_client_cmd,
         'METRICS_CLIENT_ARGS_STR': self._args_dict_to_str(
             client_pod_spec.template.metrics_args_dict),
         'POLL_INTERVAL_SECS': str(client_pod_spec.template.poll_interval_secs)
@@ -383,7 +382,7 @@ class Config:
     for image_name in docker_config_dict.keys():
       build_script_path = docker_config_dict[image_name]['buildScript']
       dockerfile_dir = docker_config_dict[image_name]['dockerFileDir']
-      build_type = docker_config_dict[image_name]['buildType']
+      build_type = docker_config_dict[image_name].get('buildType', 'opt')
       docker_images_dict[image_name] = DockerImage(gcp_project_id, image_name,
                                                    build_script_path,
                                                    dockerfile_dir, build_type)
@@ -416,11 +415,13 @@ class Config:
       temp_dict.update(templates_dict[template_name])
 
       # Create and add ClientTemplate object to the final client_templates_dict
+      stress_client_cmd = ' '.join(temp_dict['stressClientCmd'])
+      metrics_client_cmd = ' '.join(temp_dict['metricsClientCmd'])
       client_templates_dict[template_name] = ClientTemplate(
-          template_name, temp_dict['clientImagePath'],
-          temp_dict['metricsClientImagePath'], temp_dict['metricsPort'],
-          temp_dict['wrapperScriptPath'], temp_dict['pollIntervalSecs'],
-          temp_dict['clientArgs'].copy(), temp_dict['metricsArgs'].copy())
+          template_name, stress_client_cmd, metrics_client_cmd,
+          temp_dict['metricsPort'], temp_dict['wrapperScriptPath'],
+          temp_dict['pollIntervalSecs'], temp_dict['clientArgs'].copy(),
+          temp_dict['metricsArgs'].copy())
 
     return client_templates_dict
 
@@ -452,10 +453,10 @@ class Config:
       temp_dict.update(templates_dict[template_name])
 
       # Create and add ServerTemplate object to the final server_templates_dict
+      stress_server_cmd = ' '.join(temp_dict['stressServerCmd'])
       server_templates_dict[template_name] = ServerTemplate(
-          template_name, temp_dict['serverImagePath'],
-          temp_dict['wrapperScriptPath'], temp_dict['serverPort'],
-          temp_dict['serverArgs'].copy())
+          template_name, stress_server_cmd, temp_dict['wrapperScriptPath'],
+          temp_dict['serverPort'], temp_dict['serverArgs'].copy())
 
     return server_templates_dict
 
@@ -529,6 +530,8 @@ def run_tests(config):
   # run id. This is useful in debugging when looking at records in Biq query)
   run_id = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S')
   dataset_id = '%s_%s' % (config.global_settings.dataset_id_prefix, run_id)
+  print 'Run id:', run_id
+  print 'Dataset id:', dataset_id
 
   bq_helper = BigQueryHelper(run_id, '', '',
                              config.global_settings.gcp_project_id, dataset_id,