Added timeout for tests that hang

Added prettier cloud to prod output and a history of results

3 files changed, 58 insertions, 12 deletions

diff --git a/tools/gce_setup/cloud_prod_runner.sh b/tools/gce_setup/cloud_prod_runner.sh
index 520dfcd998..3a9ae51b76 100755
--- a/tools/gce_setup/cloud_prod_runner.sh
+++ b/tools/gce_setup/cloud_prod_runner.sh
@@ -28,11 +28,14 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+thisfile=$(readlink -ne "${BASH_SOURCE[0]}")
+current_time=$(date "+%Y-%m-%d-%H-%M-%S")
+result_file_name=cloud_prod_result.$current_time.html
+echo $result_file_name
 
 main() {
   source grpc_docker.sh
-  # temporarily remove ping_pong and cancel_after_first_response while investigating timeout
-  test_cases=(large_unary empty_unary client_streaming server_streaming cancel_after_begin)
+  test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response)
   auth_test_cases=(service_account_creds compute_engine_creds)
   clients=(cxx java go ruby node csharp_mono)
   for test_case in "${test_cases[@]}"
@@ -41,9 +44,9 @@ main() {
     do
       if grpc_cloud_prod_test $test_case grpc-docker-testclients $client
       then
-        echo "$test_case $client $server passed" >> /tmp/cloud_prod_result.txt
+        echo "          ['$test_case', '$client', 'prod', true]," >> /tmp/cloud_prod_result.txt
       else
-        echo "$test_case $client $server failed" >> /tmp/cloud_prod_result.txt
+        echo "          ['$test_case', '$client', 'prod', false]," >> /tmp/cloud_prod_result.txt
       fi
     done
   done
@@ -53,14 +56,20 @@ main() {
     do
       if grpc_cloud_prod_auth_test $test_case grpc-docker-testclients $client
       then
-        echo "$test_case $client $server passed" >> /tmp/cloud_prod_result.txt
+        echo "          ['$test_case', '$client', 'prod', true]," >> /tmp/cloud_prod_result.txt
       else
-        echo "$test_case $client $server failed" >> /tmp/cloud_prod_result.txt
+        echo "          ['$test_case', '$client', 'prod', false]," >> /tmp/cloud_prod_result.txt
       fi
     done
   done
-  gsutil cp /tmp/cloud_prod_result.txt gs://stoked-keyword-656-output/cloud_prod_result.txt
-  rm /tmp/cloud_prod_result.txt
+  if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    cat pre.html /tmp/cloud_prod_result.txt post.html > /tmp/cloud_prod_result.html
+    gsutil cp /tmp/cloud_prod_result.txt gs://stoked-keyword-656-output/cloud_prod_result.txt
+    gsutil cp /tmp/cloud_prod_result.html gs://stoked-keyword-656-output/cloud_prod_result.html
+    gsutil cp /tmp/cloud_prod_result.html gs://stoked-keyword-656-output/result_history/$result_file_name
+    rm /tmp/cloud_prod_result.txt
+    rm /tmp/cloud_prod_result.html
+  fi
 }
 
 set -x
diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh
index 619eff56d6..61a6afd07a 100755
--- a/tools/gce_setup/grpc_docker.sh
+++ b/tools/gce_setup/grpc_docker.sh
@@ -797,7 +797,26 @@ grpc_interop_test() {
   echo "  $ssh_cmd"
   echo "on $host"
   [[ $dry_run == 1 ]] && return 0  # don't run the command on a dry run
-  gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" 
+  gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" &
+  PID=$!
+  echo "pid is $PID"
+  for x in {0..5}
+  do
+    if ps -p $PID
+    then
+      sleep 10
+    else
+      echo "normal return before timeout"
+      # looks like the test is done
+      wait $PID
+      local ret=$?
+      echo " return $ret"
+      return $ret
+    fi
+  done
+  kill $PID
+  echo "got killed by timeout not normal"
+  return 1
 }
 
 # Runs a test command on a docker instance.
@@ -843,7 +862,26 @@ grpc_cloud_prod_test() {
   echo "  $ssh_cmd"
   echo "on $host"
   [[ $dry_run == 1 ]] && return 0  # don't run the command on a dry run
-  gcloud compute $project_opt ssh $zone_opt $host --command "$cmd"
+  gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" &
+  PID=$!
+  echo "pid is $PID"
+  for x in {0..5}
+  do
+    if ps -p $PID
+    then
+      sleep 10
+    else
+      echo "normal return before timeout"
+      # looks like the test is done
+      wait $PID
+      local ret=$?
+      echo " return $ret"
+      return $ret
+    fi
+  done
+  kill $PID
+  echo "got killed by timeout not normal"
+  return 1
 }
 
 # Runs a test command on a docker instance.
diff --git a/tools/gce_setup/interop_test_runner.sh b/tools/gce_setup/interop_test_runner.sh
index ebc631c1fd..90a78b1264 100755
--- a/tools/gce_setup/interop_test_runner.sh
+++ b/tools/gce_setup/interop_test_runner.sh
@@ -35,8 +35,7 @@ echo $result_file_name
 
 main() {
   source grpc_docker.sh
-  # temporarily remove ping_pong and cancel_after_first_response while investigating timeout
-  test_cases=(large_unary empty_unary client_streaming server_streaming cancel_after_begin)
+  test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response)
   clients=(cxx java go ruby node csharp_mono)
   servers=(cxx java go ruby node python)
   for test_case in "${test_cases[@]}"