From e8aefd216930667ff0b278d24cf411b216a9c100 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Thu, 16 Mar 2017 10:31:43 -0800 Subject: tfdbg: test examples and binaries against pip install I realized that CL/149810103 forgot to include the tfdbg binary and examples in the pip package. This CL fixes that and adds pip test for the correct inclusion of these files. Change: 150345489 --- tensorflow/python/debug/BUILD | 13 +++++ tensorflow/python/debug/examples/examples_test.sh | 70 ++++++++++++++++++----- tensorflow/tools/ci_build/builds/pip.sh | 38 ++++++++++-- 3 files changed, 101 insertions(+), 20 deletions(-) diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD index e621cf08b3..f0d245c24e 100644 --- a/tensorflow/python/debug/BUILD +++ b/tensorflow/python/debug/BUILD @@ -33,7 +33,9 @@ py_library( py_library( name = "debug_pip", deps = [ + ":debug_examples", ":debug_py", + ":offline_analyzer", ":session_debug_testlib", ], ) @@ -267,6 +269,16 @@ py_binary( ], ) +py_library( + name = "debug_examples", + deps = [ + ":debug_errors", + ":debug_fibonacci", + ":debug_mnist", + ":debug_tflearn_iris", + ], +) + py_binary( name = "debug_fibonacci", srcs = ["examples/debug_fibonacci.py"], @@ -661,6 +673,7 @@ sh_test( ":debug_fibonacci", ":debug_mnist", ":debug_tflearn_iris", + ":offline_analyzer", ], ) diff --git a/tensorflow/python/debug/examples/examples_test.sh b/tensorflow/python/debug/examples/examples_test.sh index 397078b91d..43395ac040 100755 --- a/tensorflow/python/debug/examples/examples_test.sh +++ b/tensorflow/python/debug/examples/examples_test.sh @@ -15,40 +15,80 @@ # ============================================================================== # # Bash unit tests for TensorFlow Debugger (tfdbg) Python examples that do not -# involve downloading data. +# involve downloading data. Also tests the binary offline_analyzer. +# +# Command-line flags: +# --virtualenv: (optional) If set, will test the examples and binaries +# against pip install of TensorFlow in a virtualenv. set -e +IS_VIRTUALENV=0 +PYTHON_BIN_PATH="" +while true; do + if [[ -z "$1" ]]; then + break + elif [[ "$1" == "--virtualenv" ]]; then + IS_VIRTUALENV=1 + PYTHON_BIN_PATH=$(which python) + echo + echo "IS_VIRTUALENV = ${IS_VIRTUALENV}" + echo "PYTHON_BIN_PATH = ${PYTHON_BIN_PATH}" + echo "Will test tfdbg examples and binaries against virtualenv pip install." + echo + fi + shift 1 +done -DEBUG_FIBONACCI_BIN="$TEST_SRCDIR/org_tensorflow/tensorflow/python/debug/debug_fibonacci" +if [[ -z "${PYTHON_BIN_PATH}" ]]; then + DEBUG_FIBONACCI_BIN="$TEST_SRCDIR/org_tensorflow/tensorflow/python/debug/debug_fibonacci" + DEBUG_ERRORS_BIN="$TEST_SRCDIR/org_tensorflow/tensorflow/python/debug/debug_errors" + DEBUG_MNIST_BIN="$TEST_SRCDIR/org_tensorflow/tensorflow/python/debug/debug_mnist" + DEBUG_TFLEARN_IRIS_BIN="$TEST_SRCDIR/org_tensorflow/tensorflow/python/debug/debug_tflearn_iris" + OFFLINE_ANALYZER_BIN="$TEST_SRCDIR/org_tensorflow/tensorflow/python/debug/offline_analyzer" +else + DEBUG_FIBONACCI_BIN="${PYTHON_BIN_PATH} -m tensorflow.python.debug.examples.debug_fibonacci" + DEBUG_ERRORS_BIN="${PYTHON_BIN_PATH} -m tensorflow.python.debug.examples.debug_errors" + DEBUG_MNIST_BIN="${PYTHON_BIN_PATH} -m tensorflow.python.debug.examples.debug_mnist" + DEBUG_TFLEARN_IRIS_BIN="${PYTHON_BIN_PATH} -m tensorflow.python.debug.examples.debug_tflearn_iris" + OFFLINE_ANALYZER_BIN="${PYTHON_BIN_PATH} -m tensorflow.python.debug.cli.offline_analyzer" +fi # Override the default ui_type=curses to allow the test to pass in a tty-less # test environment. -cat << EOF | "${DEBUG_FIBONACCI_BIN}" --ui_type=readline +cat << EOF | ${DEBUG_FIBONACCI_BIN} --ui_type=readline run exit EOF - -DEBUG_ERRORS_BIN="$TEST_SRCDIR/org_tensorflow/tensorflow/python/debug/debug_errors" - -cat << EOF | "${DEBUG_ERRORS_BIN}" --error=no_error --ui_type=readline +cat << EOF | ${DEBUG_ERRORS_BIN} --error=no_error --ui_type=readline run exit EOF - -DEBUG_MNIST_BIN="$TEST_SRCDIR/org_tensorflow/tensorflow/python/debug/debug_mnist" - # Use a large enough "run -t" number to let the process end properly. -cat << EOF | "${DEBUG_MNIST_BIN}" --debug --fake_data --ui_type=readline +cat << EOF | ${DEBUG_MNIST_BIN} --debug --fake_data --ui_type=readline run -f has_inf_or_nan run -t 1000 EOF - -DEBUG_TFLEARN_IRIS_BIN="$TEST_SRCDIR/org_tensorflow/tensorflow/python/debug/debug_tflearn_iris" - -cat << EOF | "${DEBUG_TFLEARN_IRIS_BIN}" --debug --fake_data --train_steps=2 --ui_type=readline +cat << EOF | ${DEBUG_TFLEARN_IRIS_BIN} --debug --fake_data --train_steps=2 --ui_type=readline run -f has_inf_or_nan EOF + +# Test offline_analyzer. +# TODO(cais): Generate an actual debug dump and load it with offline_analyzer, +# so that we can test the binary runs with a non-error exit code. +set +e +OUTPUT=$(${OFFLINE_ANALYZER_BIN} 2>&1) +set -e + +EXPECTED_OUTPUT="ERROR: dump_dir flag is empty." +if [[ "${OUTPUT}" != "${EXPECTED_OUTPUT}" ]]; then + echo "ERROR: offline_analyzer output didn't match expectation: ${OUTPUT}" 1>&2 + echo "Expected output: ${EXPECTED_OUTPUT}" + exit 1 +fi + +echo +echo "SUCCESS: tfdbg examples and binaries test PASSED" diff --git a/tensorflow/tools/ci_build/builds/pip.sh b/tensorflow/tools/ci_build/builds/pip.sh index 52acd1f732..0a78fa236b 100755 --- a/tensorflow/tools/ci_build/builds/pip.sh +++ b/tensorflow/tools/ci_build/builds/pip.sh @@ -38,6 +38,9 @@ # If NO_TEST_USER_OPS has any non-empty and non-0 value, the testing of user- # defined ops against the installation will be skipped. # +# If NO_TEST_TFDBG_BINARIES has any non-empty and non-0 value, the testing of +# TensorFlow Debugger (tfdbg) binaries and examples will be skipped. +# # Any flags not listed in the usage above will be passed directly to Bazel. # # If the --test_tutorials flag is set, it will cause the script to run the @@ -71,19 +74,26 @@ source "${SCRIPT_DIR}/builds_common.sh" CONTAINER_TYPE=$( echo "$1" | tr '[:upper:]' '[:lower:]' ) shift -if [[ ! -z "${TF_BUILD_BAZEL_CLEAN}" ]] && \ +if [[ -n "${TF_BUILD_BAZEL_CLEAN}" ]] && \ [[ "${TF_BUILD_BAZEL_CLEAN}" != "0" ]]; then echo "TF_BUILD_BAZEL_CLEAN=${TF_BUILD_BAZEL_CLEAN}: Performing 'bazel clean'" bazel clean fi DO_TEST_USER_OPS=1 -if [[ ! -z "${NO_TEST_USER_OPS}" ]] && \ +if [[ -n "${NO_TEST_USER_OPS}" ]] && \ [[ "${NO_TEST_USER_OPS}" != "0" ]]; then echo "NO_TEST_USER_OPS=${NO_TEST_USER_OPS}: Will skip testing of user ops" DO_TEST_USER_OPS=0 fi +DO_TEST_TFDBG_BINARIES=1 +if [[ -n "${NO_TEST_TFDBG_BINARIES}" ]] && \ + [[ "${NO_TEST_TFDBG_BINARIES}" != "0" ]]; then + echo "NO_TEST_TFDBG_BINARIES=${NO_TEST_TFDBG_BINARIES}: Will skip testing of tfdbg binaries" + DO_TEST_TFDBG_BINARIES=0 +fi + DO_TEST_TUTORIALS=0 DO_INTEGRATION_TESTS=0 BAZEL_FLAGS="" @@ -127,7 +137,7 @@ fi # If still in a virtualenv, deactivate it first -if [[ ! -z "$(which deactivate)" ]]; then +if [[ -n "$(which deactivate)" ]]; then echo "It appears that we are already in a virtualenv. Deactivating..." deactivate || die "FAILED: Unable to deactivate from existing virtualenv" fi @@ -188,7 +198,7 @@ fi WHL_DIR=$(dirname "${WHL_PATH}") WHL_BASE_NAME=$(basename "${WHL_PATH}") -if [[ ! -z "${PY_TAGS}" ]]; then +if [[ -n "${PY_TAGS}" ]]; then NEW_WHL_BASE_NAME=$(echo ${WHL_BASE_NAME} | cut -d \- -f 1)-\ $(echo ${WHL_BASE_NAME} | cut -d \- -f 2)-${PY_TAGS}-${PLATFORM_TAG}.whl @@ -270,7 +280,7 @@ for PACKAGE in ${INSTALL_EXTRA_PIP_PACKAGES}; do die "pip install ${PACKAGE} FAILED" done -if [[ ! -z "${NO_TEST_ON_INSTALL}" ]] && +if [[ -n "${NO_TEST_ON_INSTALL}" ]] && [[ "${NO_TEST_ON_INSTALL}" != "0" ]]; then echo "NO_TEST_ON_INSTALL=${NO_TEST_ON_INSTALL}:" echo " Skipping ALL Python unit tests on install" @@ -286,6 +296,24 @@ if [[ "${DO_TEST_USER_OPS}" == "1" ]]; then die "PIP user-op tests-on-install FAILED" fi +# Test TensorFlow Debugger (tfdbg) examples. +if [[ "${DO_TEST_TFDBG_BINARIES}" == "1" ]]; then + echo + echo "Testing TensorFlow Debugger (tfdbg) binaries" + echo + + # cd to a temporary directory to avoid picking up Python files in the source + # tree. + TMP_DIR=$(mktemp -d) + pushd "${TMP_DIR}" + + "${SCRIPT_DIR}/../../../python/debug/examples/examples_test.sh" \ + --virtualenv || \ + die "PIP tests-on-install of tfdbg binaries FAILED" + + popd +fi + # Optional: Run the tutorial tests if [[ "${DO_TEST_TUTORIALS}" == "1" ]]; then "${SCRIPT_DIR}/test_tutorials.sh" --virtualenv || \ -- cgit v1.2.3