diff options
Diffstat (limited to 'tensorflow/tools/ci_build')
11 files changed, 49 insertions, 55 deletions
diff --git a/tensorflow/tools/ci_build/builds/pip.sh b/tensorflow/tools/ci_build/builds/pip.sh index 85c712d3c6..e0a1391d6e 100755 --- a/tensorflow/tools/ci_build/builds/pip.sh +++ b/tensorflow/tools/ci_build/builds/pip.sh @@ -205,17 +205,14 @@ if [[ -n "${PY_TAGS}" ]]; then $(echo ${WHL_BASE_NAME} | cut -d \- -f 2)-${PY_TAGS}-${PLATFORM_TAG}.whl if [[ ! -f "${WHL_DIR}/${NEW_WHL_BASE_NAME}" ]]; then - if cp "${WHL_DIR}/${WHL_BASE_NAME}" "${WHL_DIR}/${NEW_WHL_BASE_NAME}" - then - echo "Copied wheel file: ${WHL_BASE_NAME} --> ${NEW_WHL_BASE_NAME}" - else + cp "${WHL_DIR}/${WHL_BASE_NAME}" "${WHL_DIR}/${NEW_WHL_BASE_NAME}" && \ + echo "Copied wheel file: ${WHL_BASE_NAME} --> ${NEW_WHL_BASE_NAME}" || \ die "ERROR: Failed to copy wheel file to ${NEW_WHL_BASE_NAME}" - fi fi fi if [[ $(uname) == "Linux" ]]; then - AUDITED_WHL_NAME="${WHL_DIR}/$(echo ${WHL_BASE_NAME//linux/manylinux1})" + AUDITED_WHL_NAME="${WHL_DIR}/$(echo ${WHL_BASE_NAME} | sed "s/linux/manylinux1/")" # Repair the wheels for cpu manylinux1 if [[ ${CONTAINER_TYPE} == "cpu" ]]; then @@ -243,20 +240,14 @@ echo "Installing pip whl file: ${WHL_PATH}" VENV_DIR="${PIP_TEST_ROOT}/venv" if [[ -d "${VENV_DIR}" ]]; then - if rm -rf "${VENV_DIR}" - then - echo "Removed existing virtualenv directory: ${VENV_DIR}" - else - die "Failed to remove existing virtualenv directory: ${VENV_DIR}" - fi + rm -rf "${VENV_DIR}" && \ + echo "Removed existing virtualenv directory: ${VENV_DIR}" || \ + die "Failed to remove existing virtualenv directory: ${VENV_DIR}" fi -if mkdir -p ${VENV_DIR} -then - echo "Created virtualenv directory: ${VENV_DIR}" -else - die "FAILED to create virtualenv directory: ${VENV_DIR}" -fi +mkdir -p ${VENV_DIR} && \ + echo "Created virtualenv directory: ${VENV_DIR}" || \ + die "FAILED to create virtualenv directory: ${VENV_DIR}" # Verify that virtualenv exists if [[ -z $(which virtualenv) ]]; then diff --git a/tensorflow/tools/ci_build/ci_build.sh b/tensorflow/tools/ci_build/ci_build.sh index 9c1b75d004..3b640dd5e8 100755 --- a/tensorflow/tools/ci_build/ci_build.sh +++ b/tensorflow/tools/ci_build/ci_build.sh @@ -80,7 +80,7 @@ fi # cmake (CPU) builds do not require configuration. if [[ "${CONTAINER_TYPE}" == "cmake" ]]; then - CI_COMMAND_PREFIX=("") + CI_COMMAND_PREFIX="" fi # Use nvidia-docker if the container is GPU. @@ -120,9 +120,9 @@ DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | tr '[:upper:]' '[:lower:]') # Print arguments. echo "WORKSPACE: ${WORKSPACE}" -echo "CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[*]}" -echo "COMMAND: ${COMMAND[*]}" -echo "CI_COMMAND_PREFIX: ${CI_COMMAND_PREFIX[*]}" +echo "CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[@]}" +echo "COMMAND: ${COMMAND[@]}" +echo "CI_COMMAND_PREFIX: ${CI_COMMAND_PREFIX[@]}" echo "CONTAINER_TYPE: ${CONTAINER_TYPE}" echo "BUILD_TAG: ${BUILD_TAG}" echo " (docker container name will be ${DOCKER_IMG_NAME})" @@ -140,7 +140,7 @@ if [[ $? != "0" ]]; then fi # Run the command inside the container. -echo "Running '${COMMAND[*]}' inside ${DOCKER_IMG_NAME}..." +echo "Running '${COMMAND[@]}' inside ${DOCKER_IMG_NAME}..." mkdir -p ${WORKSPACE}/bazel-ci_build-cache # By default we cleanup - remove the container once it finish running (--rm) # and share the PID namespace (--pid=host) so the process inside does not have diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh index 1cf87d7c7c..dfaf50eb4f 100755 --- a/tensorflow/tools/ci_build/ci_parameterized_build.sh +++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh @@ -200,8 +200,8 @@ echo " TF_BUILD_ENABLE_XLA=${TF_BUILD_ENABLE_XLA}" function get_cuda_capability_version() { if [[ ! -z $(which deviceQuery) ]]; then # The first listed device is used - deviceQuery | grep "CUDA Capability .* version" | \ - head -1 | awk '{print $NF}' + echo $(deviceQuery | grep "CUDA Capability .* version" | \ + head -1 | awk '{print $NF}') fi } @@ -532,14 +532,11 @@ if [[ "${TF_BUILD_PYTHON_VERSION}" == "python3.5" ]]; then DOCKERFILE="${TMP_DIR}/Dockerfile.${TF_BUILD_CONTAINER_TYPE}" # Replace a line in the Dockerfile - if sed -i \ + sed -i \ 's/RUN \/install\/install_pip_packages.sh/RUN \/install\/install_python3.5_pip_packages.sh/g' \ - "${DOCKERFILE}" - then - echo "Copied and modified Dockerfile for Python 3.5 build: ${DOCKERFILE}" - else - die "ERROR: Faild to copy and modify Dockerfile: ${DOCKERFILE}" - fi + "${DOCKERFILE}" && \ + echo "Copied and modified Dockerfile for Python 3.5 build: ${DOCKERFILE}" || \ + die "ERROR: Faild to copy and modify Dockerfile: ${DOCKERFILE}" DOCKERFILE_FLAG="--dockerfile ${DOCKERFILE}" fi @@ -577,7 +574,7 @@ rm -f ${TMP_SCRIPT} END_TIME=$(date +'%s') echo "" echo "Parameterized build ends with ${RESULT} at: $(date) "\ -"(Elapsed time: $((END_TIME - START_TIME)) s)" +"(Elapsed time: $((${END_TIME} - ${START_TIME})) s)" # Clean up temporary directory if it exists diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh index e428766a40..fd2874df91 100755 --- a/tensorflow/tools/ci_build/ci_sanity.sh +++ b/tensorflow/tools/ci_build/ci_sanity.sh @@ -47,7 +47,7 @@ num_cpus() { # Get the hash of the last non-merge git commit on the current branch. # Usage: get_last_non_merge_git_commit get_last_non_merge_git_commit() { - git rev-list --no-merges -n 1 HEAD + echo $(git rev-list --no-merges -n 1 HEAD) } # List files changed (i.e., added, removed or revised) in the last non-merge @@ -75,7 +75,7 @@ get_py_files_to_check() { echo "${PY_FILES}" else - find tensorflow -name '*.py' + echo $(find tensorflow -name '*.py') fi } @@ -157,25 +157,25 @@ do_pylint() { NONWL_ERRORS_FILE="$(mktemp)_pylint_nonwl_errors.log" rm -rf ${OUTPUT_FILE} - rm -rf ${ERRORS_FILE} + rm -rf ${ERRORS_FLIE} rm -rf ${NONWL_ERRORS_FILE} touch ${NONWL_ERRORS_FILE} ${PYLINT_BIN} --rcfile="${PYLINTRC_FILE}" --output-format=parseable \ - --jobs=${NUM_CPUS} ${PYTHON_SRC_FILES} > ${OUTPUT_FILE} 2>&1 + --jobs=${NUM_CPUS} ${PYTHON_SRC_FILES} 2>&1 > ${OUTPUT_FILE} PYLINT_END_TIME=$(date +'%s') echo "" - echo "pylint took $((PYLINT_END_TIME - PYLINT_START_TIME)) s" + echo "pylint took $((${PYLINT_END_TIME} - ${PYLINT_START_TIME})) s" echo "" grep -E '(\[E|\[W0311|\[W0312)' ${OUTPUT_FILE} > ${ERRORS_FILE} N_ERRORS=0 - while read -r LINE; do + while read LINE; do IS_WHITELISTED=0 for WL_REGEX in ${ERROR_WHITELIST}; do - if echo ${LINE} | grep -q "${WL_REGEX}"; then + if [[ ! -z $(echo ${LINE} | grep "${WL_REGEX}") ]]; then echo "Found a whitelisted error:" echo " ${LINE}" IS_WHITELISTED=1 @@ -248,7 +248,7 @@ do_pep8() { PEP8_END_TIME=$(date +'%s') echo "" - echo "pep8 took $((PEP8_END_TIME - PEP8_START_TIME)) s" + echo "pep8 took $((${PEP8_END_TIME} - ${PEP8_START_TIME})) s" echo "" if [[ -s ${PEP8_OUTPUT_FILE} ]]; then @@ -278,7 +278,7 @@ do_buildifier(){ BUILDIFIER_END_TIME=$(date +'%s') echo "" - echo "buildifier took $((BUILDIFIER_END_TIME - BUILDIFIER_START_TIME)) s" + echo "buildifier took $((${BUILDIFIER_END_TIME} - ${BUILDIFIER_START_TIME})) s" echo "" if [[ -s ${BUILDIFIER_OUTPUT_FILE} ]]; then @@ -306,7 +306,7 @@ do_external_licenses_check(){ echo "Getting external dependencies for ${BUILD_TARGET}" bazel query "attr('licenses', 'notice', deps(${BUILD_TARGET}))" --no_implicit_deps --no_host_deps --keep_going \ - | grep -E -v "^//tensorflow" \ + | egrep -v "^//tensorflow" \ | sed -e 's|:.*||' \ | sort \ | uniq 2>&1 \ @@ -315,7 +315,7 @@ do_external_licenses_check(){ echo echo "Getting list of external licenses mentioned in ${LICENSES_TARGET}." bazel query "deps(${LICENSES_TARGET})" --no_implicit_deps --no_host_deps --keep_going \ - | grep -E -v "^//tensorflow" \ + | egrep -v "^//tensorflow" \ | sed -e 's|:.*||' \ | sort \ | uniq 2>&1 \ @@ -329,7 +329,7 @@ do_external_licenses_check(){ EXTERNAL_LICENSES_CHECK_END_TIME=$(date +'%s') echo - echo "do_external_licenses_check took $((EXTERNAL_LICENSES_CHECK_END_TIME - EXTERNAL_LICENSES_CHECK_START_TIME)) s" + echo "do_external_licenses_check took $((${EXTERNAL_LICENSES_CHECK_END_TIME} - ${EXTERNAL_LICENSES_CHECK_START_TIME})) s" echo if [[ -s ${MISSING_LICENSES_FILE} ]] || [[ -s ${EXTRA_LICENSES_FILE} ]] ; then diff --git a/tensorflow/tools/ci_build/install/install_pip_packages.sh b/tensorflow/tools/ci_build/install/install_pip_packages.sh index 7fcd235e62..b8f9fc8453 100755 --- a/tensorflow/tools/ci_build/install/install_pip_packages.sh +++ b/tensorflow/tools/ci_build/install/install_pip_packages.sh @@ -44,8 +44,8 @@ pip2 install --upgrade markdown==2.6.8 pip3 install --upgrade markdown==2.6.8 # Install protobuf. -pip2 install --upgrade protobuf==3.3.0 -pip3 install --upgrade protobuf==3.3.0 +pip2 install --upgrade protobuf==3.2.0 +pip3 install --upgrade protobuf==3.2.0 # Remove obsolete version of six, which can sometimes confuse virtualenv. rm -rf /usr/lib/python3/dist-packages/six* diff --git a/tensorflow/tools/ci_build/install/install_proto3.sh b/tensorflow/tools/ci_build/install/install_proto3.sh index 7934002b2c..773c89b70b 100755 --- a/tensorflow/tools/ci_build/install/install_proto3.sh +++ b/tensorflow/tools/ci_build/install/install_proto3.sh @@ -17,9 +17,9 @@ # Install protobuf3. # Select protobuf version. -PROTOBUF_VERSION="3.3.0" +PROTOBUF_VERSION="3.2.0" protobuf_ver_flat=$(echo $PROTOBUF_VERSION | sed 's/\.//g' | sed 's/^0*//g') -local_protobuf_ver=$(protoc --version) +local_protobuf_ver=$(protoc --version | awk '{print $2}') local_protobuf_ver_flat=$(echo $local_protobuf_ver | sed 's/\.//g' | sed 's/^0*//g') if [[ -z $local_protobuf_ver_flat ]]; then local_protobuf_ver_flat=0 @@ -30,7 +30,7 @@ if (( $local_protobuf_ver_flat < $protobuf_ver_flat )); then PROTOBUF_ZIP=$(basename "${PROTOBUF_URL}") UNZIP_DEST="google-protobuf" - wget "${PROTOBUF_URL}" + wget -q "${PROTOBUF_URL}" unzip "${PROTOBUF_ZIP}" -d "${UNZIP_DEST}" cp "${UNZIP_DEST}/bin/protoc" /usr/local/bin/ diff --git a/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh b/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh index 084ac49496..e7e2d256cd 100755 --- a/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh +++ b/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh @@ -64,7 +64,7 @@ set -e pip3.5 install --upgrade six==1.10.0 # Install protobuf. -pip3.5 install --upgrade protobuf==3.3.0 +pip3.5 install --upgrade protobuf==3.2.0 # Remove obsolete version of six, which can sometimes confuse virtualenv. rm -rf /usr/lib/python3/dist-packages/six* diff --git a/tensorflow/tools/ci_build/protobuf/protobuf_optimized_pip.sh b/tensorflow/tools/ci_build/protobuf/protobuf_optimized_pip.sh index 3e31aa1ce1..59ba71f5df 100755 --- a/tensorflow/tools/ci_build/protobuf/protobuf_optimized_pip.sh +++ b/tensorflow/tools/ci_build/protobuf/protobuf_optimized_pip.sh @@ -14,7 +14,7 @@ # limitations under the License. # ============================================================================== -PROTOBUF_VERSION="3.3.1" +PROTOBUF_VERSION="3.2.0" PYTHON_BIN=${PYTHON_BIN:-python} DIR=${PWD}/protobuf diff --git a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh index dff4707cbe..5581023ad7 100644 --- a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh +++ b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh @@ -161,5 +161,5 @@ function create_python_test_dir() { function reinstall_tensorflow_pip() { echo "y" | pip uninstall tensorflow -q || true - pip install ${1} --no-deps + pip install ${1} } diff --git a/tensorflow/tools/ci_build/windows/cpu/cmake/run_py.bat b/tensorflow/tools/ci_build/windows/cpu/cmake/run_py.bat index 96fbadd176..8b6836f4ed 100644 --- a/tensorflow/tools/ci_build/windows/cpu/cmake/run_py.bat +++ b/tensorflow/tools/ci_build/windows/cpu/cmake/run_py.bat @@ -29,6 +29,9 @@ IF DEFINED PIP_EXE (ECHO PIP_EXE is set to %PIP_EXE%) ELSE (SET PIP_EXE="C:\Prog CALL %REPO_ROOT%\tensorflow\tools\ci_build\windows\cpu\cmake\run_build.bat if %errorlevel% neq 0 exit /b %errorlevel% +:: Attempt to upgrade PIP to work around Anaconda issue #542. +%PIP_EXE% install --ignore-installed --upgrade pip setuptools -v -v + :: Since there are no wildcards in windows command prompt, use dark magic to get the wheel file name. DIR %REPO_ROOT%\%BUILD_DIR%\tf_python\dist\ /S /B > wheel_filename_file set /p WHEEL_FILENAME=<wheel_filename_file @@ -36,7 +39,7 @@ del wheel_filename_file :: Install the pip package. echo Installing PIP package... -%PIP_EXE% install --upgrade --no-deps %WHEEL_FILENAME% -v -v +%PIP_EXE% install --upgrade %WHEEL_FILENAME% -v -v if %errorlevel% neq 0 exit /b %errorlevel% :: Run all python tests if the installation succeeded. diff --git a/tensorflow/tools/ci_build/windows/gpu/cmake/run_py.bat b/tensorflow/tools/ci_build/windows/gpu/cmake/run_py.bat index e774a6e916..ba2d939b5f 100644 --- a/tensorflow/tools/ci_build/windows/gpu/cmake/run_py.bat +++ b/tensorflow/tools/ci_build/windows/gpu/cmake/run_py.bat @@ -28,6 +28,9 @@ IF DEFINED PIP_EXE (ECHO PIP_EXE is set to %PIP_EXE%) ELSE (SET PIP_EXE="C:\Prog CALL %REPO_ROOT%\tensorflow\tools\ci_build\windows\gpu\cmake\run_build.bat if %errorlevel% neq 0 exit /b %errorlevel% +:: Attempt to upgrade PIP to work around Anaconda issue #542. +%PIP_EXE% install --ignore-installed --upgrade pip setuptools -v -v + :: Since there are no wildcards in windows command prompt, use dark magic to get the wheel file name. DIR %REPO_ROOT%\%BUILD_DIR%\tf_python\dist\ /S /B > wheel_filename_file set /p WHEEL_FILENAME=<wheel_filename_file @@ -35,7 +38,7 @@ del wheel_filename_file :: Install the pip package. echo Installing PIP package... -%PIP_EXE% install --upgrade --no-deps %WHEEL_FILENAME% -v -v +%PIP_EXE% install --upgrade %WHEEL_FILENAME% -v -v if %errorlevel% neq 0 exit /b %errorlevel% :: Run all python tests if the installation succeeded. |