diff options
author | Martin Wicke <wicke@google.com> | 2016-07-25 13:48:16 -0800 |
---|---|---|
committer | TensorFlower Gardener <gardener@tensorflow.org> | 2016-07-25 15:02:34 -0700 |
commit | 21716d8f6e175cd6e8cd97a84e48497574268b0c (patch) | |
tree | 3345202e7a812cc9572beb24fc01732696a4140d /tensorflow/tools/gcs_test | |
parent | ed281973d66d0030e58a77a05821bbb88627f5bd (diff) |
Merge changes from github.
Change: 128401884
Diffstat (limited to 'tensorflow/tools/gcs_test')
-rw-r--r-- | tensorflow/tools/gcs_test/Dockerfile | 6 | ||||
-rwxr-xr-x | tensorflow/tools/gcs_test/gcs_smoke.sh | 24 | ||||
-rwxr-xr-x | tensorflow/tools/gcs_test/gcs_smoke_wrapper.sh | 98 |
3 files changed, 103 insertions, 25 deletions
diff --git a/tensorflow/tools/gcs_test/Dockerfile b/tensorflow/tools/gcs_test/Dockerfile index 43e7da7743..be3ad40b15 100644 --- a/tensorflow/tools/gcs_test/Dockerfile +++ b/tensorflow/tools/gcs_test/Dockerfile @@ -16,7 +16,9 @@ RUN ./install_google_cloud_sdk.bash --disable-prompts --install-dir=/var/gcloud # Install nightly TensorFlow pip RUN pip install \ - http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.9.0-py2-none-any.whl + http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-0.9.0-cp27-none-linux_x86_64.whl # Copy test files -COPY python/gcs_smoke.py / +RUN mkdir -p /gcs-smoke/python +COPY gcs_smoke_wrapper.sh /gcs-smoke/ +COPY python/gcs_smoke.py /gcs-smoke/python/ diff --git a/tensorflow/tools/gcs_test/gcs_smoke.sh b/tensorflow/tools/gcs_test/gcs_smoke.sh index df59781ea8..6deff2e919 100755 --- a/tensorflow/tools/gcs_test/gcs_smoke.sh +++ b/tensorflow/tools/gcs_test/gcs_smoke.sh @@ -67,30 +67,8 @@ docker build --no-cache \ # Run the docker image with the GCS key file mapped and the gcloud-required # environment variables set. -LOG_FILE="/tmp/tf-gcs-test.log" -rm -rf ${LOG_FILE} - docker run --rm \ -v ${GCLOUD_JSON_KEY_PATH}:/gcloud-key.json \ -e "GOOGLE_APPLICATION_CREDENTIALS=/gcloud-key.json" \ "${DOCKER_IMG}" \ - python /gcs_smoke.py --gcs_bucket_url="${GCS_BUCKET_URL}" \ - 2>&1 > "${LOG_FILE}" - -if [[ $? != "0" ]]; then - cat ${LOG_FILE} - die "FAIL: End-to-end test of GCS access from TensorFlow failed." -fi - -cat ${LOG_FILE} -echo "" - -# Clean up the newly created tfrecord file in GCS bucket -NEW_TFREC_URL=$(grep "Using input path" "${LOG_FILE}" | \ - awk '{print $NF}') -if [[ -z ${NEW_TFREC_URL} ]]; then - die "FAIL: Unable to determine the URL to the new tfrecord file in GCS" -fi -gsutil rm "${NEW_TFREC_URL}" && \ - echo "Cleaned up new tfrecord file in GCS: ${NEW_TFREC_URL}" || \ - die "FAIL: Unable to clean up new tfrecord file in GCS: ${NEW_TFREC_URL}"
\ No newline at end of file + /gcs-smoke/gcs_smoke_wrapper.sh "${GCS_BUCKET_URL}" diff --git a/tensorflow/tools/gcs_test/gcs_smoke_wrapper.sh b/tensorflow/tools/gcs_test/gcs_smoke_wrapper.sh new file mode 100755 index 0000000000..89a0dd9169 --- /dev/null +++ b/tensorflow/tools/gcs_test/gcs_smoke_wrapper.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# +# In-container wrapper for GCS smoke test. +# +# This script invokes gcs_smoke.py and performs tear down afterwards. +# +# Usage: +# gcs_smoke_wrapper.sh <GCS_BUCKET_URL> + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Helper function: Exit on failure. +die () { + echo $@ + exit 1 +} + +print_usage() { + echo "Usage: gcs_smoke_wrapper.sh <GCS_BUCKET_URL>" + echo "" +} + +# Sanity check on command-line arguments. +GCS_BUCKET_URL=$1 +if [[ -z "${GCS_BUCKET_URL}" ]]; then + print_usage + die "ERROR: Command-line argument GCS_BUCKET_URL is not supplied" +fi + +# Check that gcloud and gsutil binaries are available. +GCLOUD_BIN="/var/gcloud/google-cloud-sdk/bin/gcloud" +if [[ ! -f "${GCLOUD_BIN}" ]]; then + die "ERROR: Unable to find gcloud at path ${GCLOUD_BIN}" +fi + +GSUTIL_BIN="/var/gcloud/google-cloud-sdk/bin/gsutil" +if [[ ! -f "${GSUTIL_BIN}" ]]; then + die "ERROR: Unable to find gsutil at path ${GSUTIL_BIN}" +fi + +# Check environment variable for gcloud credentials +if [[ -z "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then + die "ERROR: Required gcloud environment variable "\ +"${GOOGLE_APPLICATION_CREDENTIALS} is not set." +fi + +# Locate main Python file +GCS_SMOKE_PY="${SCRIPT_DIR}/python/gcs_smoke.py" +if [[ ! -f "${GCS_SMOKE_PY}" ]]; then + die "ERROR: Unable to find Python file at ${GCS_SMOKE_PY}" +fi + + +LOG_FILE="/tmp/tf-gcs-test.log" +rm -rf ${LOG_FILE} || \ + die "ERROR: Failed to remove existing log file ${LOG_FILE}" + +# Invoke main Python file +python "${GCS_SMOKE_PY}" --gcs_bucket_url="${GCS_BUCKET_URL}" \ + 2>&1 > "${LOG_FILE}" + +if [[ $? != "0" ]]; then + cat ${LOG_FILE} + die "FAIL: End-to-end test of GCS access from TensorFlow failed." +fi + +cat ${LOG_FILE} +echo "" + + +# Clean up the newly created tfrecord file in GCS bucket. +# First, activate gcloud service account +"${GCLOUD_BIN}" auth activate-service-account \ + --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" || \ + die "ERROR: Failed to activate gcloud service account with JSON key file" + +NEW_TFREC_URL=$(grep "Using input path" "${LOG_FILE}" | \ + awk '{print $NF}') +if [[ -z ${NEW_TFREC_URL} ]]; then + die "FAIL: Unable to determine the URL to the new tfrecord file in GCS" +fi +"${GSUTIL_BIN}" rm "${NEW_TFREC_URL}" && \ + echo "Cleaned up new tfrecord file in GCS: ${NEW_TFREC_URL}" || \ + die "FAIL: Unable to clean up new tfrecord file in GCS: ${NEW_TFREC_URL}"
\ No newline at end of file |