diff options
Diffstat (limited to 'tensorflow/tools/ci_build')
24 files changed, 236 insertions, 181 deletions
diff --git a/tensorflow/tools/ci_build/Dockerfile.cmake b/tensorflow/tools/ci_build/Dockerfile.cmake index e8c3199828..4587bcf891 100644 --- a/tensorflow/tools/ci_build/Dockerfile.cmake +++ b/tensorflow/tools/ci_build/Dockerfile.cmake @@ -28,8 +28,8 @@ RUN pip install --upgrade astor RUN pip install --upgrade gast RUN pip install --upgrade numpy RUN pip install --upgrade termcolor -RUN pip install keras_applications==1.0.2 -RUN pip install keras_preprocessing==1.0.1 +RUN pip install keras_applications==1.0.4 +RUN pip install keras_preprocessing==1.0.2 # Install golang RUN apt-get install -t xenial-backports -y golang-1.9 diff --git a/tensorflow/tools/ci_build/Dockerfile.cpu.ppc64le b/tensorflow/tools/ci_build/Dockerfile.cpu.ppc64le index e879c34bbd..ada2c63880 100644 --- a/tensorflow/tools/ci_build/Dockerfile.cpu.ppc64le +++ b/tensorflow/tools/ci_build/Dockerfile.cpu.ppc64le @@ -7,7 +7,7 @@ COPY install/*.sh /install/ RUN /install/install_bootstrap_deb_packages.sh RUN add-apt-repository -y ppa:openjdk-r/ppa RUN /install/install_deb_packages.sh -RUN apt-get update && apt-get install -y libopenblas-dev +RUN /install/install_openblas_ppc64le.sh RUN /install/install_hdf5_ppc64le.sh RUN /install/install_pip_packages.sh RUN /install/install_bazel_from_source.sh diff --git a/tensorflow/tools/ci_build/Dockerfile.gpu b/tensorflow/tools/ci_build/Dockerfile.gpu index 7591ecc04e..383f9545c9 100644 --- a/tensorflow/tools/ci_build/Dockerfile.gpu +++ b/tensorflow/tools/ci_build/Dockerfile.gpu @@ -14,6 +14,7 @@ RUN /install/install_bootstrap_deb_packages.sh RUN add-apt-repository -y ppa:openjdk-r/ppa && \ add-apt-repository -y ppa:george-edison55/cmake-3.x RUN /install/install_deb_packages.sh + RUN /install/install_pip_packages.sh RUN /install/install_bazel.sh RUN /install/install_golang.sh @@ -22,6 +23,11 @@ RUN /install/install_golang.sh COPY install/.bazelrc /etc/bazel.bazelrc ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH +# Link NCCL libray and header where the build script expects them. +RUN mkdir /usr/local/cuda-9.0/lib && \ + ln -s /usr/lib/x86_64-linux-gnu/libnccl.so.2 /usr/local/cuda/lib/libnccl.so.2 && \ + ln -s /usr/include/nccl.h /usr/local/cuda/include/nccl.h + # Configure the build for our CUDA configuration. ENV TF_NEED_CUDA 1 ENV TF_CUDA_COMPUTE_CAPABILITIES 3.0 diff --git a/tensorflow/tools/ci_build/Dockerfile.gpu.ppc64le b/tensorflow/tools/ci_build/Dockerfile.gpu.ppc64le index 8967138747..a404f129ab 100644 --- a/tensorflow/tools/ci_build/Dockerfile.gpu.ppc64le +++ b/tensorflow/tools/ci_build/Dockerfile.gpu.ppc64le @@ -13,7 +13,7 @@ ARG DEBIAN_FRONTEND=noninteractive RUN /install/install_bootstrap_deb_packages.sh RUN add-apt-repository -y ppa:openjdk-r/ppa RUN /install/install_deb_packages.sh -RUN apt-get update && apt-get install -y libopenblas-dev +RUN /install/install_openblas_ppc64le.sh RUN /install/install_hdf5_ppc64le.sh RUN /install/install_pip_packages.sh RUN /install/install_bazel_from_source.sh diff --git a/tensorflow/tools/ci_build/builds/android.sh b/tensorflow/tools/ci_build/builds/android.sh index d81793efe0..7c3e308229 100755 --- a/tensorflow/tools/ci_build/builds/android.sh +++ b/tensorflow/tools/ci_build/builds/android.sh @@ -26,13 +26,19 @@ configure_android_workspace # android_full.sh echo "========== TensorFlow Demo Build Test ==========" +TARGETS= +TARGETS+=" //tensorflow/examples/android:tensorflow_demo" +# Also build the Eager Runtime so it remains compatible with Android for the +# benefits of clients like TensorFlow Lite. For now it is enough to build only +# :execute, which what TF Lite needs. +TARGETS+=" //tensorflow/core/common_runtime/eager:execute" # Enable sandboxing so that zip archives don't get incorrectly packaged # in assets/ dir (see https://github.com/bazelbuild/bazel/issues/2334) # TODO(gunan): remove extra flags once sandboxing is enabled for all builds. bazel --bazelrc=/dev/null build \ --compilation_mode=opt --cxxopt=-std=c++11 --fat_apk_cpu=x86_64 \ --spawn_strategy=sandboxed --genrule_strategy=sandboxed \ - //tensorflow/examples/android:tensorflow_demo + ${TARGETS} echo "========== Makefile Build Test ==========" # Test Makefile build just to make sure it still works. diff --git a/tensorflow/tools/ci_build/builds/pip.sh b/tensorflow/tools/ci_build/builds/pip.sh index 883bb93647..fef121ab5a 100755 --- a/tensorflow/tools/ci_build/builds/pip.sh +++ b/tensorflow/tools/ci_build/builds/pip.sh @@ -314,7 +314,10 @@ create_activate_virtualenv_and_install_tensorflow() { # Upgrade pip so it supports tags such as cp27mu, manylinux1 etc. echo "Upgrade pip in virtualenv" - pip install --upgrade pip==9.0.1 + + # NOTE: pip install --upgrade pip leads to a documented TLS issue for + # some versions in python + curl https://bootstrap.pypa.io/get-pip.py | python # Force tensorflow reinstallation. Otherwise it may not get installed from # last build if it had the same version number as previous build. diff --git a/tensorflow/tools/ci_build/builds/run_pip_tests.sh b/tensorflow/tools/ci_build/builds/run_pip_tests.sh index 29680e6882..bbaf59c69a 100755 --- a/tensorflow/tools/ci_build/builds/run_pip_tests.sh +++ b/tensorflow/tools/ci_build/builds/run_pip_tests.sh @@ -97,7 +97,8 @@ fi # TF_BUILD_APPEND_ARGUMENTS any user supplied args. BAZEL_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py \ --build_tests_only -k --test_tag_filters=${PIP_TEST_FILTER_TAG} \ - --test_timeout 300,450,1200,3600 ${TF_BUILD_APPEND_ARGUMENTS}" + --test_timeout 300,450,1200,3600 ${TF_BUILD_APPEND_ARGUMENTS} \ + --test_output=errors" BAZEL_TEST_TARGETS="//${PIP_TEST_PREFIX}/tensorflow/contrib/... \ //${PIP_TEST_PREFIX}/tensorflow/python/... \ diff --git a/tensorflow/tools/ci_build/ci_build.sh b/tensorflow/tools/ci_build/ci_build.sh index f6a50d3d4c..77265e0f50 100755 --- a/tensorflow/tools/ci_build/ci_build.sh +++ b/tensorflow/tools/ci_build/ci_build.sh @@ -115,6 +115,7 @@ DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | tr '[:upper:]' '[:lower:]') # Print arguments. echo "WORKSPACE: ${WORKSPACE}" +echo "CI_DOCKER_BUILD_EXTRA_PARAMS: ${CI_DOCKER_BUILD_EXTRA_PARAMS[*]}" echo "CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[*]}" echo "COMMAND: ${COMMAND[*]}" echo "CI_COMMAND_PREFIX: ${CI_COMMAND_PREFIX[*]}" @@ -126,7 +127,7 @@ echo "" # Build the docker container. echo "Building container (${DOCKER_IMG_NAME})..." -docker build -t ${DOCKER_IMG_NAME} \ +docker build -t ${DOCKER_IMG_NAME} ${CI_DOCKER_BUILD_EXTRA_PARAMS[@]} \ -f "${DOCKERFILE_PATH}" "${DOCKER_CONTEXT_PATH}" # Check docker build status diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh index d49d4b0c49..993894d658 100755 --- a/tensorflow/tools/ci_build/ci_parameterized_build.sh +++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh @@ -131,7 +131,7 @@ BAZEL_CMD="bazel test" BAZEL_BUILD_ONLY_CMD="bazel build" BAZEL_CLEAN_CMD="bazel clean" -DEFAULT_BAZEL_CONFIGS="--config=gcp --config=hdfs" +DEFAULT_BAZEL_CONFIGS="" PIP_CMD="${CI_BUILD_DIR}/builds/pip.sh" PIP_TEST_TUTORIALS_FLAG="--test_tutorials" @@ -150,36 +150,7 @@ BAZEL_TARGET="//tensorflow/... -//tensorflow/compiler/..." if [[ -n "$TF_SKIP_CONTRIB_TESTS" ]]; then BAZEL_TARGET="$BAZEL_TARGET -//tensorflow/contrib/..." else - BAZEL_TARGET="${BAZEL_TARGET} -//tensorflow/contrib/lite/..." - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite:context_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite:framework" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite:interpreter_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite:model_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/toco:toco" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite:simple_memory_arena_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite:string_util_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:activations_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:add_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:basic_rnn_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:concatenation_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:conv_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:depthwise_conv_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:embedding_lookup_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:embedding_lookup_sparse_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:fully_connected_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:hashtable_lookup_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:local_response_norm_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:lsh_projection_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:lstm_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:l2norm_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:mul_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:pooling_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:reshape_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:resize_bilinear_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:skip_gram_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:softmax_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:space_to_depth_test" - BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:svdf_test" + BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/..." fi TUT_TEST_DATA_DIR="/tmp/tf_tutorial_test_data" @@ -570,33 +541,35 @@ echo "" TMP_DIR="" DOCKERFILE_FLAG="" -if [[ "${TF_BUILD_PYTHON_VERSION}" == "python3.5" ]] || - [[ "${TF_BUILD_PYTHON_VERSION}" == "python3.6" ]]; then - # Modify Dockerfile for Python3.5 | Python3.6 build - TMP_DIR=$(mktemp -d) - echo "Docker build will occur in temporary directory: ${TMP_DIR}" - - # Copy the files required for the docker build - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - cp -r "${SCRIPT_DIR}/install" "${TMP_DIR}/install" || \ - die "ERROR: Failed to copy directory ${SCRIPT_DIR}/install" - - DOCKERFILE="${SCRIPT_DIR}/Dockerfile.${TF_BUILD_CONTAINER_TYPE}" - cp "${DOCKERFILE}" "${TMP_DIR}/" || \ - die "ERROR: Failed to copy Dockerfile at ${DOCKERFILE}" - DOCKERFILE="${TMP_DIR}/Dockerfile.${TF_BUILD_CONTAINER_TYPE}" - - # Replace a line in the Dockerfile - if sed -i \ - "s/RUN \/install\/install_pip_packages.sh/RUN \/install\/install_${TF_BUILD_PYTHON_VERSION}_pip_packages.sh/g" \ - "${DOCKERFILE}" - then - echo "Copied and modified Dockerfile for ${TF_BUILD_PYTHON_VERSION} build: ${DOCKERFILE}" - else - die "ERROR: Faild to copy and modify Dockerfile: ${DOCKERFILE}" - fi +if [[ "${DO_DOCKER}" == "1" ]]; then + if [[ "${TF_BUILD_PYTHON_VERSION}" == "python3.5" ]] || + [[ "${TF_BUILD_PYTHON_VERSION}" == "python3.6" ]]; then + # Modify Dockerfile for Python3.5 | Python3.6 build + TMP_DIR=$(mktemp -d) + echo "Docker build will occur in temporary directory: ${TMP_DIR}" + + # Copy the files required for the docker build + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + cp -r "${SCRIPT_DIR}/install" "${TMP_DIR}/install" || \ + die "ERROR: Failed to copy directory ${SCRIPT_DIR}/install" + + DOCKERFILE="${SCRIPT_DIR}/Dockerfile.${TF_BUILD_CONTAINER_TYPE}" + cp "${DOCKERFILE}" "${TMP_DIR}/" || \ + die "ERROR: Failed to copy Dockerfile at ${DOCKERFILE}" + DOCKERFILE="${TMP_DIR}/Dockerfile.${TF_BUILD_CONTAINER_TYPE}" + + # Replace a line in the Dockerfile + if sed -i \ + "s/RUN \/install\/install_pip_packages.sh/RUN \/install\/install_${TF_BUILD_PYTHON_VERSION}_pip_packages.sh/g" \ + "${DOCKERFILE}" + then + echo "Copied and modified Dockerfile for ${TF_BUILD_PYTHON_VERSION} build: ${DOCKERFILE}" + else + die "ERROR: Faild to copy and modify Dockerfile: ${DOCKERFILE}" + fi - DOCKERFILE_FLAG="--dockerfile ${DOCKERFILE}" + DOCKERFILE_FLAG="--dockerfile ${DOCKERFILE}" + fi fi chmod +x ${TMP_SCRIPT} diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh index f0a437c183..866fe95d2b 100755 --- a/tensorflow/tools/ci_build/ci_sanity.sh +++ b/tensorflow/tools/ci_build/ci_sanity.sh @@ -354,7 +354,7 @@ do_external_licenses_check(){ # Whitelist echo ${EXTRA_LICENSE_FILE} - grep -e "@bazel_tools//src" -e "@bazel_tools//tools/" -e "@com_google_absl//" -e "//external" -e "@local" -e "@com_github_googlecloudplatform_google_cloud_cpp//" -v ${EXTRA_LICENSES_FILE} > temp.txt + grep -e "@bazel_tools//src" -e "@bazel_tools//tools/" -e "@com_google_absl//" -e "//external" -e "@local" -e "@com_github_googlecloudplatform_google_cloud_cpp//" -e "@embedded_jdk//" -v ${EXTRA_LICENSES_FILE} > temp.txt mv temp.txt ${EXTRA_LICENSES_FILE} @@ -543,7 +543,7 @@ SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_check_futures_test" "d SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "Check that python files have certain __future__ imports" "buildifier check" "bazel nobuild" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies" "Pip Smoke Test: Checking py_test dependencies exist in pip package" "Check load py_test: Check that BUILD files with py_test target properly load py_test" "Code Link Check: Check there are no broken links" "Test entries in /tensorflow/contrib/cmake/python_{modules|protos|protos_cc}.txt for validity and consistency" "Check file names for cases") INCREMENTAL_FLAG="" -DEFAULT_BAZEL_CONFIGS="--config=hdfs --config=gcp" +DEFAULT_BAZEL_CONFIGS="" # Parse command-line arguments BAZEL_FLAGS=${DEFAULT_BAZEL_CONFIGS} diff --git a/tensorflow/tools/ci_build/gpu_build/parallel_gpu_execute.sh b/tensorflow/tools/ci_build/gpu_build/parallel_gpu_execute.sh index d0816c92b7..75da9bb835 100755 --- a/tensorflow/tools/ci_build/gpu_build/parallel_gpu_execute.sh +++ b/tensorflow/tools/ci_build/gpu_build/parallel_gpu_execute.sh @@ -35,6 +35,30 @@ elif [[ ${BASH_VER_MAJOR} -eq 4 ]] && [[ ${BASH_VER_MINOR} -lt 2 ]]; then exit 1 fi +function is_absolute { + [[ "$1" = /* ]] || [[ "$1" =~ ^[a-zA-Z]:[/\\].* ]] +} + +RUNFILES_MANIFEST_FILE="${TEST_SRCDIR}/MANIFEST" +function rlocation() { + if is_absolute "$1" ; then + # If the file path is already fully specified, simply return it. + echo "$1" + elif [[ -e "$TEST_SRCDIR/$1" ]]; then + # If the file exists in the $TEST_SRCDIR then just use it. + echo "$TEST_SRCDIR/$1" + elif [[ -e "$RUNFILES_MANIFEST_FILE" ]]; then + # If a runfiles manifest file exists then use it. + echo "$(grep "^$1 " "$RUNFILES_MANIFEST_FILE" | sed 's/[^ ]* //')" + fi +} + +TEST_BINARY="$(rlocation $TEST_WORKSPACE/${1#./})" +shift + +# Make sure /var/lock exists, this may not be true under MSYS +mkdir -p /var/lock + TF_GPU_COUNT=${TF_GPU_COUNT:-8} for i in `seq 0 $((TF_GPU_COUNT-1))`; do @@ -45,8 +69,8 @@ for i in `seq 0 $((TF_GPU_COUNT-1))`; do # This export only works within the brackets, so it is isolated to one # single command. export CUDA_VISIBLE_DEVICES=$i - echo "Running test $* on GPU $CUDA_VISIBLE_DEVICES" - $@ + echo "Running test $TEST_BINARY $* on GPU $CUDA_VISIBLE_DEVICES" + "$TEST_BINARY" $@ ) return_code=$? flock -u "$lock_fd" diff --git a/tensorflow/tools/ci_build/install/install_bazel.sh b/tensorflow/tools/ci_build/install/install_bazel.sh index 3e27a94cf2..e284401b8a 100755 --- a/tensorflow/tools/ci_build/install/install_bazel.sh +++ b/tensorflow/tools/ci_build/install/install_bazel.sh @@ -15,7 +15,7 @@ # ============================================================================== # Select bazel version. -BAZEL_VERSION="0.11.0" +BAZEL_VERSION="0.15.0" set +e local_bazel_ver=$(bazel version 2>&1 | grep -i label | awk '{print $3}') diff --git a/tensorflow/tools/ci_build/install/install_bazel_from_source.sh b/tensorflow/tools/ci_build/install/install_bazel_from_source.sh index ddad00c5f0..87be81577d 100755 --- a/tensorflow/tools/ci_build/install/install_bazel_from_source.sh +++ b/tensorflow/tools/ci_build/install/install_bazel_from_source.sh @@ -18,7 +18,7 @@ # It will compile bazel from source and install it in /usr/local/bin # Select bazel version. -BAZEL_VERSION="0.11.0" +BAZEL_VERSION="0.15.0" set +e local_bazel_ver=$(bazel version 2>&1 | grep -i label | awk '{print $3}') diff --git a/tensorflow/tools/ci_build/install/install_openblas_ppc64le.sh b/tensorflow/tools/ci_build/install/install_openblas_ppc64le.sh new file mode 100755 index 0000000000..107cc61ff5 --- /dev/null +++ b/tensorflow/tools/ci_build/install/install_openblas_ppc64le.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +OPENBLAS_SRC_PATH=/tmp/openblas_src/ +POWER="POWER8" +USE_OPENMP="USE_OPENMP=1" +OPENBLAS_INSTALL_PATH="/usr" +apt-get update +apt-get install -y gfortran gfortran-5 +rm -rf ${OPENBLAS_SRC_PATH} +git clone -b release-0.3.0 https://github.com/xianyi/OpenBLAS ${OPENBLAS_SRC_PATH} +cd ${OPENBLAS_SRC_PATH} +# Pick up fix for OpenBLAS issue 1571 +git cherry-pick -X theirs 961d25e9c7e4a1758adb1dbeaa15187de69dd052 +make TARGET=${POWER} ${USE_OPENMP} FC=gfortran +make PREFIX=${OPENBLAS_INSTALL_PATH} install diff --git a/tensorflow/tools/ci_build/install/install_pip_packages.sh b/tensorflow/tools/ci_build/install/install_pip_packages.sh index 221b5b80fb..bb316ecfc9 100755 --- a/tensorflow/tools/ci_build/install/install_pip_packages.sh +++ b/tensorflow/tools/ci_build/install/install_pip_packages.sh @@ -61,11 +61,11 @@ rm -rf /usr/lib/python3/dist-packages/six* # https://github.com/tensorflow/tensorflow/issues/6968 # This workaround isn't needed for Ubuntu 16.04 or later. if $(cat /etc/*-release | grep -q 14.04); then - pip2 install --no-binary=:all: --upgrade numpy==1.12.0 - pip3 install --no-binary=:all: --upgrade numpy==1.12.0 + pip2 install --no-binary=:all: --upgrade numpy==1.14.5 + pip3 install --no-binary=:all: --upgrade numpy==1.14.5 else - pip2 install --upgrade numpy==1.12.0 - pip3 install --upgrade numpy==1.12.0 + pip2 install --upgrade numpy==1.14.5 + pip3 install --upgrade numpy==1.14.5 fi pip2 install scipy==0.18.1 @@ -115,10 +115,10 @@ pip2 install --upgrade setuptools==39.1.0 pip3 install --upgrade setuptools==39.1.0 # Keras -pip2 install keras_applications==1.0.2 -pip3 install keras_applications==1.0.2 -pip2 install keras_preprocessing==1.0.1 -pip3 install keras_preprocessing==1.0.1 +pip2 install keras_applications==1.0.4 --no-deps +pip3 install keras_applications==1.0.4 --no-deps +pip2 install keras_preprocessing==1.0.2 --no-deps +pip3 install keras_preprocessing==1.0.2 --no-deps # Install last working version of setuptools. pip2 install --upgrade setuptools==39.1.0 diff --git a/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh b/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh index 45a30c6e82..15e4396ce3 100755 --- a/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh +++ b/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh @@ -58,7 +58,7 @@ rm -rf /usr/lib/python3/dist-packages/six* # numpy needs to be installed from source to fix segfaults. See: # https://github.com/tensorflow/tensorflow/issues/6968 # This workaround isn't needed for Ubuntu 16.04 or later. -pip3.5 install --no-binary=:all: --upgrade numpy==1.12.0 +pip3.5 install --no-binary=:all: --upgrade numpy==1.14.5 pip3.5 install scipy==0.18.1 @@ -85,8 +85,8 @@ pip3.5 install --upgrade termcolor pip3.5 install --upgrade setuptools==39.1.0 # Keras -pip3.5 install keras_applications==1.0.2 -pip3.5 install keras_preprocessing==1.0.1 +pip3.5 install keras_applications==1.0.4 +pip3.5 install keras_preprocessing==1.0.2 # Install last working version of setuptools. pip3.5 install --upgrade setuptools==39.1.0 diff --git a/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh b/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh index d66b2aa18a..0fc3eee71c 100755 --- a/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh +++ b/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh @@ -70,7 +70,7 @@ rm -rf /usr/lib/python3/dist-packages/six* # numpy needs to be installed from source to fix segfaults. See: # https://github.com/tensorflow/tensorflow/issues/6968 # This workaround isn't needed for Ubuntu 16.04 or later. -pip3 install --no-binary=:all: --upgrade numpy==1.12.0 +pip3 install --no-binary=:all: --upgrade numpy==1.14.5 pip3 install scipy==0.18.1 @@ -101,7 +101,7 @@ pip3 install --upgrade termcolor pip3 install --upgrade setuptools==39.1.0 # Keras -pip3.5 install keras_applications==1.0.2 -pip3.5 install keras_preprocessing==1.0.1 +pip3 install keras_applications==1.0.4 +pip3 install keras_preprocessing==1.0.2 # LINT.ThenChange(//tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh) diff --git a/tensorflow/tools/ci_build/linux/cpu/run_py3_contrib.sh b/tensorflow/tools/ci_build/linux/cpu/run_py3_contrib.sh index 2b68de3c5b..f6fa9251d4 100755 --- a/tensorflow/tools/ci_build/linux/cpu/run_py3_contrib.sh +++ b/tensorflow/tools/ci_build/linux/cpu/run_py3_contrib.sh @@ -34,35 +34,4 @@ yes "" | $PYTHON_BIN_PATH configure.py bazel test --test_tag_filters=-no_oss,-oss_serial,-gpu,-benchmark-test -k \ --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 --config=opt \ --test_size_filters=small,medium --test_output=errors -- \ - //tensorflow/contrib/... \ - -//tensorflow/contrib/lite/... \ - //tensorflow/contrib/lite:context_test \ - //tensorflow/contrib/lite:framework \ - //tensorflow/contrib/lite:interpreter_test \ - //tensorflow/contrib/lite:model_test \ - //tensorflow/contrib/lite/toco:toco \ - //tensorflow/contrib/lite:simple_memory_arena_test \ - //tensorflow/contrib/lite:string_util_test \ - //tensorflow/contrib/lite/kernels:activations_test \ - //tensorflow/contrib/lite/kernels:add_test \ - //tensorflow/contrib/lite/kernels:basic_rnn_test \ - //tensorflow/contrib/lite/kernels:concatenation_test \ - //tensorflow/contrib/lite/kernels:conv_test \ - //tensorflow/contrib/lite/kernels:depthwise_conv_test \ - //tensorflow/contrib/lite/kernels:embedding_lookup_test \ - //tensorflow/contrib/lite/kernels:embedding_lookup_sparse_test \ - //tensorflow/contrib/lite/kernels:fully_connected_test \ - //tensorflow/contrib/lite/testing:generated_zip_tests \ - //tensorflow/contrib/lite/kernels:hashtable_lookup_test \ - //tensorflow/contrib/lite/kernels:local_response_norm_test \ - //tensorflow/contrib/lite/kernels:lsh_projection_test \ - //tensorflow/contrib/lite/kernels:lstm_test \ - //tensorflow/contrib/lite/kernels:l2norm_test \ - //tensorflow/contrib/lite/kernels:mul_test \ - //tensorflow/contrib/lite/kernels:pooling_test \ - //tensorflow/contrib/lite/kernels:reshape_test \ - //tensorflow/contrib/lite/kernels:resize_bilinear_test \ - //tensorflow/contrib/lite/kernels:skip_gram_test \ - //tensorflow/contrib/lite/kernels:softmax_test \ - //tensorflow/contrib/lite/kernels:space_to_depth_test \ - //tensorflow/contrib/lite/kernels:svdf_test + //tensorflow/contrib/... diff --git a/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh b/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh index ad22ebe4eb..a1d91a6123 100755 --- a/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh +++ b/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh @@ -34,12 +34,17 @@ echo "TF_DOCKER_BUILD_DEVEL_BRANCH=${TF_DOCKER_BUILD_DEVEL_BRANCH}" echo "TF_DOCKER_BUILD_IMAGE_NAME=${TF_DOCKER_BUILD_IMAGE_NAME}" echo "TF_DOCKER_BUILD_VERSION=${TF_DOCKER_BUILD_VERSION}" +# Build containers for AVX +# Include the instructions for sandybridge and later, but tune for ivybridge +TF_BAZEL_BUILD_OPTIONS="--config=mkl --copt=-march=sandybridge --copt=-mtune=ivybridge --copt=-O3 --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0" + # build the python 2 container and whl TF_DOCKER_BUILD_TYPE="MKL" \ TF_DOCKER_BUILD_IS_DEVEL="YES" \ TF_DOCKER_BUILD_DEVEL_BRANCH="${TF_DOCKER_BUILD_DEVEL_BRANCH}" \ TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \ TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}" \ + TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \ ${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh # build the python 3 container and whl @@ -49,5 +54,29 @@ TF_DOCKER_BUILD_TYPE="MKL" \ TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \ TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}" \ TF_DOCKER_BUILD_PYTHON_VERSION="PYTHON3" \ + TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \ + ${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh + +# Build containers for AVX2 +# Include the instructions for haswell and later, but tune for broadwell +TF_BAZEL_BUILD_OPTIONS="--config=mkl --copt=-march=haswell --copt=-mtune=broadwell --copt=-O3 --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0" + +# build the python 2 container and whl +TF_DOCKER_BUILD_TYPE="MKL" \ + TF_DOCKER_BUILD_IS_DEVEL="YES" \ + TF_DOCKER_BUILD_DEVEL_BRANCH="${TF_DOCKER_BUILD_DEVEL_BRANCH}" \ + TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \ + TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}-avx2" \ + TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \ ${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh +# build the python 3 container and whl +TF_DOCKER_BUILD_TYPE="MKL" \ + TF_DOCKER_BUILD_IS_DEVEL="YES" \ + TF_DOCKER_BUILD_DEVEL_BRANCH="${TF_DOCKER_BUILD_DEVEL_BRANCH}" \ + TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \ + TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}-avx2" \ + TF_DOCKER_BUILD_PYTHON_VERSION="PYTHON3" \ + TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \ + ${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh + diff --git a/tensorflow/tools/ci_build/update_version.py b/tensorflow/tools/ci_build/update_version.py index 642dde36a7..30c318a58f 100755 --- a/tensorflow/tools/ci_build/update_version.py +++ b/tensorflow/tools/ci_build/update_version.py @@ -248,16 +248,6 @@ def update_md_files(old_version, new_version): replace_string_in_line(r"<version>%s<\/version>" % old_version, "<version>%s</version>" % new_version, filepath) - # Update any links to colab notebooks. - def colab_url(version): - version_string = "%s.%s.%s" % (version.major, version.minor, version.patch) - prefix = "https://colab.research.google.com/github/tensorflow/models/blob/r" - return prefix + version_string + "/" - - replace_string_in_line( - colab_url(old_version), colab_url(new_version), - "%s/docs_src/get_started/eager.md" % TF_SRC_DIR) - def major_minor_change(old_version, new_version): """Check if a major or minor change occurred.""" diff --git a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh index a3e07737a4..0482cf619a 100644 --- a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh +++ b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh @@ -23,17 +23,20 @@ function run_configure_for_gpu_build { # Enable CUDA support export TF_NEED_CUDA=1 - # TODO(pcloudy): Remove this after TensorFlow uses its own CRSOOTOOL - # for GPU build on Windows - export USE_MSVC_WRAPPER=1 - yes "" | ./configure } -function set_gcs_remote_cache_options { - echo "build --experimental_remote_spawn_cache" >> "${TMP_BAZELRC}" +function set_remote_cache_options { + echo "build --remote_instance_name=projects/tensorflow-testing-cpu" >> "${TMP_BAZELRC}" echo "build --experimental_remote_platform_override='properties:{name:\"build\" value:\"windows-x64\"}'" >> "${TMP_BAZELRC}" - echo "build --remote_http_cache=https://storage.googleapis.com/$GCS_BUCKET_NAME" >> "${TMP_BAZELRC}" + echo "build --remote_cache=remotebuildexecution.googleapis.com" >> "${TMP_BAZELRC}" + echo "build --tls_enabled=true" >> "${TMP_BAZELRC}" + echo "build --remote_timeout=3600" >> "${TMP_BAZELRC}" + echo "build --auth_enabled=true" >> "${TMP_BAZELRC}" + echo "build --spawn_strategy=standalone" >> "${TMP_BAZELRC}" + echo "build --strategy=Javac=standalone" >> "${TMP_BAZELRC}" + echo "build --strategy=Closure=standalone" >> "${TMP_BAZELRC}" + echo "build --genrule_strategy=standalone" >> "${TMP_BAZELRC}" echo "build --google_credentials=$GOOGLE_CLOUD_CREDENTIAL" >> "${TMP_BAZELRC}" } diff --git a/tensorflow/tools/ci_build/windows/bazel/common_env.sh b/tensorflow/tools/ci_build/windows/bazel/common_env.sh index 8a237e4e28..333a89d3f5 100644 --- a/tensorflow/tools/ci_build/windows/bazel/common_env.sh +++ b/tensorflow/tools/ci_build/windows/bazel/common_env.sh @@ -26,7 +26,8 @@ # * Bazel windows executable copied as "bazel.exe" and included in PATH. # Use a temporary directory with a short name. -export TMPDIR="C:/tmp" +export TMPDIR=${TMPDIR:-"C:/tmp"} +export TMPDIR=$(cygpath -m "$TMPDIR") mkdir -p "$TMPDIR" # Set bash path @@ -54,10 +55,10 @@ export PATH="/c/${PYTHON_BASE_PATH}/Scripts:$PATH" export TF_CUDA_VERSION=${TF_CUDA_VERSION:-9.0} export TF_CUDNN_VERSION=${TF_CUDNN_VERSION:-7.0} export TF_CUDA_COMPUTE_CAPABILITIES=${TF_CUDA_COMPUTE_CAPABILITIES:-3.7} -export CUDA_INSTALL_PATH=${CUDA_INSTALL_PATH:-"C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${TF_CUDA_VERSION}"} +export CUDA_TOOLKIT_PATH=${CUDA_TOOLKIT_PATH:-"C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${TF_CUDA_VERSION}"} export CUDNN_INSTALL_PATH=${CUDNN_INSTALL_PATH:-"C:/tools/cuda"} # Add Cuda and Cudnn dll directories into PATH -export PATH="$(cygpath -u "${CUDA_INSTALL_PATH}")/bin:$PATH" -export PATH="$(cygpath -u "${CUDA_INSTALL_PATH}")/extras/CUPTI/libx64:$PATH" +export PATH="$(cygpath -u "${CUDA_TOOLKIT_PATH}")/bin:$PATH" +export PATH="$(cygpath -u "${CUDA_TOOLKIT_PATH}")/extras/CUPTI/libx64:$PATH" export PATH="$(cygpath -u "${CUDNN_INSTALL_PATH}")/bin:$PATH" diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh index 5c305f7512..47e0e5dd59 100644 --- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh +++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh @@ -53,30 +53,39 @@ function cleanup { } trap cleanup EXIT -skip_test=0 -release_build=0 +PY_TEST_DIR="py_test_dir" +SKIP_TEST=0 +RELEASE_BUILD=0 +TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/... \ + //${PY_TEST_DIR}/tensorflow/contrib/... " + +# --skip_test Skip running tests +# --enable_remote_cache Add options to enable remote cache for build and test +# --release_build Build for release, compilation time will be longer to +# ensure performance +# --test_core_only Use tensorflow/python/... as test target +# --test_contrib_only Use tensorflow/contrib/... as test target for ARG in "$@"; do - if [[ "$ARG" == --skip_test ]]; then - skip_test=1 - elif [[ "$ARG" == --enable_gcs_remote_cache ]]; then - set_gcs_remote_cache_options - elif [[ "$ARG" == --release_build ]]; then - release_build=1 - fi + case "$ARG" in + --skip_test) SKIP_TEST=1 ;; + --enable_remote_cache) set_remote_cache_options ;; + --release_build) RELEASE_BUILD=1 ;; + --test_core_only) TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/..." ;; + --test_contrib_only) TEST_TARGET="//${PY_TEST_DIR}/tensorflow/contrib/..." ;; + *) + esac done -if [[ "$release_build" != 1 ]]; then - # --define=override_eigen_strong_inline=true speeds up the compiling of conv_grad_ops_3d.cc and conv_ops_3d.cc +if [[ "$RELEASE_BUILD" == 1 ]]; then + # Overriding eigen strong inline speeds up the compiling of conv_grad_ops_3d.cc and conv_ops_3d.cc # by 20 minutes. See https://github.com/tensorflow/tensorflow/issues/10521 - # Because this hurts the performance of TF, we don't enable it in release build. - echo "build --define=override_eigen_strong_inline=true" >> "${TMP_BAZELRC}" + # Because this hurts the performance of TF, we don't override it in release build. + export TF_OVERRIDE_EIGEN_STRONG_INLINE=0 +else + export TF_OVERRIDE_EIGEN_STRONG_INLINE=1 fi -# The host and target platforms are the same in Windows build. So we don't have -# to distinct them. This helps avoid building the same targets twice. -echo "build --distinct_host_configuration=false" >> "${TMP_BAZELRC}" - # Enable short object file path to avoid long path issue on Windows. echo "startup --output_user_root=${TMPDIR}" >> "${TMP_BAZELRC}" @@ -88,12 +97,11 @@ run_configure_for_cpu_build bazel build --announce_rc --config=opt tensorflow/tools/pip_package:build_pip_package || exit $? -if [[ "$skip_test" == 1 ]]; then +if [[ "$SKIP_TEST" == 1 ]]; then exit 0 fi # Create a python test directory to avoid package name conflict -PY_TEST_DIR="py_test_dir" create_python_test_dir "${PY_TEST_DIR}" ./bazel-bin/tensorflow/tools/pip_package/build_pip_package "$PWD/${PY_TEST_DIR}" @@ -111,7 +119,7 @@ bazel test --announce_rc --config=opt -k --test_output=errors \ --define=no_tensorflow_py_deps=true --test_lang_filters=py \ --test_tag_filters=-no_pip,-no_windows,-no_oss \ --build_tag_filters=-no_pip,-no_windows,-no_oss --build_tests_only \ + --test_size_filters=small,medium \ --jobs="${N_JOBS}" --test_timeout="300,450,1200,3600" \ --flaky_test_attempts=3 \ - //${PY_TEST_DIR}/tensorflow/python/... \ - //${PY_TEST_DIR}/tensorflow/contrib/... + ${TEST_TARGET} diff --git a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh index ededad615a..e3eee11080 100644 --- a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh +++ b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh @@ -53,30 +53,39 @@ function cleanup { } trap cleanup EXIT -skip_test=0 -release_build=0 +PY_TEST_DIR="py_test_dir" +SKIP_TEST=0 +RELEASE_BUILD=0 +TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/... \ + //${PY_TEST_DIR}/tensorflow/contrib/... " + +# --skip_test Skip running tests +# --enable_remote_cache Add options to enable remote cache for build and test +# --release_build Build for release, compilation time will be longer to +# ensure performance +# --test_core_only Use tensorflow/python/... as test target +# --test_contrib_only Use tensorflow/contrib/... as test target for ARG in "$@"; do - if [[ "$ARG" == --skip_test ]]; then - skip_test=1 - elif [[ "$ARG" == --enable_gcs_remote_cache ]]; then - set_gcs_remote_cache_options - elif [[ "$ARG" == --release_build ]]; then - release_build=1 - fi + case "$ARG" in + --skip_test) SKIP_TEST=1 ;; + --enable_remote_cache) set_remote_cache_options ;; + --release_build) RELEASE_BUILD=1 ;; + --test_core_only) TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/..." ;; + --test_contrib_only) TEST_TARGET="//${PY_TEST_DIR}/tensorflow/contrib/..." ;; + *) + esac done -if [[ "$release_build" != 1 ]]; then - # --define=override_eigen_strong_inline=true speeds up the compiling of conv_grad_ops_3d.cc and conv_ops_3d.cc +if [[ "$RELEASE_BUILD" == 1 ]]; then + # Overriding eigen strong inline speeds up the compiling of conv_grad_ops_3d.cc and conv_ops_3d.cc # by 20 minutes. See https://github.com/tensorflow/tensorflow/issues/10521 - # Because this hurts the performance of TF, we don't enable it in release build. - echo "build --define=override_eigen_strong_inline=true" >> "${TMP_BAZELRC}" + # Because this hurts the performance of TF, we don't override it in release build. + export TF_OVERRIDE_EIGEN_STRONG_INLINE=0 +else + export TF_OVERRIDE_EIGEN_STRONG_INLINE=1 fi -# The host and target platforms are the same in Windows build. So we don't have -# to distinct them. This helps avoid building the same targets twice. -echo "build --distinct_host_configuration=false" >> "${TMP_BAZELRC}" - # Enable short object file path to avoid long path issue on Windows. echo "startup --output_user_root=${TMPDIR}" >> "${TMP_BAZELRC}" @@ -91,12 +100,11 @@ run_configure_for_gpu_build bazel build --announce_rc --config=opt tensorflow/tools/pip_package:build_pip_package || exit $? -if [[ "$skip_test" == 1 ]]; then +if [[ "$SKIP_TEST" == 1 ]]; then exit 0 fi # Create a python test directory to avoid package name conflict -PY_TEST_DIR="py_test_dir" create_python_test_dir "${PY_TEST_DIR}" ./bazel-bin/tensorflow/tools/pip_package/build_pip_package "$PWD/${PY_TEST_DIR}" @@ -105,14 +113,18 @@ create_python_test_dir "${PY_TEST_DIR}" PIP_NAME=$(ls ${PY_TEST_DIR}/tensorflow-*.whl) reinstall_tensorflow_pip ${PIP_NAME} +TF_GPU_COUNT=${TF_GPU_COUNT:-8} + # Define no_tensorflow_py_deps=true so that every py_test has no deps anymore, # which will result testing system installed tensorflow # GPU tests are very flaky when running concurrently, so set local_test_jobs=1 bazel test --announce_rc --config=opt -k --test_output=errors \ + --test_env=TF_GPU_COUNT \ + --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ --define=no_tensorflow_py_deps=true --test_lang_filters=py \ --test_tag_filters=-no_pip,-no_windows,-no_windows_gpu,-no_gpu,-no_pip_gpu,-no_oss \ --build_tag_filters=-no_pip,-no_windows,-no_windows_gpu,-no_gpu,-no_pip_gpu,-no_oss --build_tests_only \ - --local_test_jobs=1 --test_timeout="300,450,1200,3600" \ + --test_size_filters=small,medium \ + --local_test_jobs=$TF_GPU_COUNT --test_timeout="300,450,1200,3600" \ --flaky_test_attempts=3 \ - //${PY_TEST_DIR}/tensorflow/python/... \ - //${PY_TEST_DIR}/tensorflow/contrib/... + ${TEST_TARGET} |