diff options
author | gunan <gunan@google.com> | 2016-11-30 15:23:13 -0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2016-11-30 15:23:13 -0800 |
commit | fb04d0f2833a5feebdef06320cad0443a0b8de19 (patch) | |
tree | 385305c1218bfd8144304dbb2c34fc4808cc538b | |
parent | a70a2d9f0e7d34e36018e83e4b17c45a26a2f3dc (diff) | |
parent | 68172c8e584fb1ffd85287e674231bddca78a10e (diff) |
Merge pull request #5961 from meteorcloudy/run-test-windows
Run cc tests on Windows with Bazel
6 files changed, 212 insertions, 25 deletions
diff --git a/tensorflow/core/debug/debug_io_utils.cc b/tensorflow/core/debug/debug_io_utils.cc index dc7121e6c3..ec20485dc0 100644 --- a/tensorflow/core/debug/debug_io_utils.cc +++ b/tensorflow/core/debug/debug_io_utils.cc @@ -18,6 +18,9 @@ limitations under the License. #include <vector> #include "grpc++/create_channel.h" +// winsock2.h is used in grpc, so Ws2_32.lib is needed +#pragma comment(lib,"Ws2_32.lib") + #include "tensorflow/core/debug/debug_service.grpc.pb.h" #include "tensorflow/core/framework/summary.pb.h" #include "tensorflow/core/lib/io/path.h" diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl index f827ae3858..ed504e9db2 100644 --- a/tensorflow/core/platform/default/build_config.bzl +++ b/tensorflow/core/platform/default/build_config.bzl @@ -152,7 +152,16 @@ def tf_additional_test_deps(): return [] def tf_additional_test_srcs(): - return ["platform/default/test_benchmark.cc", "platform/posix/test.cc"] + return [ + "platform/default/test_benchmark.cc", + ] + select({ + "//tensorflow:windows" : [ + "platform/windows/test.cc" + ], + "//conditions:default" : [ + "platform/posix/test.cc", + ], + }) def tf_kernel_tests_linkstatic(): return 0 diff --git a/tensorflow/tools/ci_build/windows/cpu/bazel/common_env.sh b/tensorflow/tools/ci_build/windows/cpu/bazel/common_env.sh new file mode 100644 index 0000000000..6e7e555065 --- /dev/null +++ b/tensorflow/tools/ci_build/windows/cpu/bazel/common_env.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# +# This script assumes the standard setup on tensorflow Jenkins windows machines. +# It is NOT guaranteed to work on any other machine. Use at your own risk! +# +# REQUIREMENTS: +# * All installed in standard locations: +# - JDK8, and JAVA_HOME set. +# - Microsoft Visual Studio 2015 Community Edition +# - Msys2 +# - Anaconda3 +# * Bazel windows executable copied as "bazel.exe" and included in PATH. + +# All commands shall pass, and all should be visible. +set -x +set -e + +# Use a temporary directory with a short name. +export TMPDIR="C:/tmp" +mkdir -p "$TMPDIR" + +# Set bash path +export BAZEL_SH="C:/tools/msys64/usr/bin/bash" + +# Set Python path for ./configure +export PYTHON_BIN_PATH="C:/Program Files/Anaconda3/python" + +# Set Python path for cc_configure.bzl +export BAZEL_PYTHON="C:/Program Files/Anaconda3/python" + +# Set Visual Studio path +export BAZEL_VS="C:/Program Files (x86)/Microsoft Visual Studio 14.0" + +# Add python into PATH, it's needed because gen_git_source.py uses +# '/usr/bin/env python' as a shebang +export PATH="/c/Program Files/Anaconda3:$PATH" diff --git a/tensorflow/tools/ci_build/windows/cpu/bazel/run_cc_test_windows.bat b/tensorflow/tools/ci_build/windows/cpu/bazel/run_cc_test_windows.bat new file mode 100644 index 0000000000..99aea4278b --- /dev/null +++ b/tensorflow/tools/ci_build/windows/cpu/bazel/run_cc_test_windows.bat @@ -0,0 +1 @@ +c:\tools\msys64\usr\bin\bash -l %cd%/tensorflow/tools/ci_build/windows/cpu/bazel/run_cc_test_windows.sh %* diff --git a/tensorflow/tools/ci_build/windows/cpu/bazel/run_cc_test_windows.sh b/tensorflow/tools/ci_build/windows/cpu/bazel/run_cc_test_windows.sh new file mode 100644 index 0000000000..3e882656a9 --- /dev/null +++ b/tensorflow/tools/ci_build/windows/cpu/bazel/run_cc_test_windows.sh @@ -0,0 +1,140 @@ +#!/bin/bash +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# +# This script assumes the standard setup on tensorflow Jenkins windows machines. +# It is NOT guaranteed to work on any other machine. Use at your own risk! +# +# REQUIREMENTS: +# * All installed in standard locations: +# - JDK8, and JAVA_HOME set. +# - Microsoft Visual Studio 2015 Community Edition +# - Msys2 +# - Anaconda3 +# * Bazel windows executable copied as "bazel.exe" and included in PATH. + +# All commands shall pass, and all should be visible. +set -x +set -e + +# This script is under <repo_root>/tensorflow/tools/ci_build/windows/cpu/bazel +# Change into repository root. +script_dir=$(dirname $0) +cd ${script_dir%%tensorflow/tools/ci_build/windows/cpu/bazel}. + +# Setting up the environment variables Bazel and ./configure needs +source "tensorflow/tools/ci_build/windows/cpu/bazel/common_env.sh" \ + || { echo "Failed to source common_env.sh" >&2; exit 1; } + +# bazel clean --expunge doesn't work on Windows yet. +# Clean the output base manually to ensure build correctness +bazel clean +output_base=$(bazel info output_base) +bazel shutdown +# Sleep 5s to wait for jvm shutdown completely +# otherwise rm will fail with device or resource busy error +sleep 5 +rm -rf ${output_base} + +export TF_NEED_CUDA=0 +echo "" | ./configure + +failing_tests="\ + //tensorflow/core:example_example_parser_configuration_test + \ + //tensorflow/core/kernels:sparse_dense_binary_op_shared_test + \ + //tensorflow/core/kernels:sparse_reduce_sum_op_test + \ + //tensorflow/core:lib_core_status_test + \ + //tensorflow/core:lib_monitoring_collection_registry_test + \ + //tensorflow/core:lib_strings_numbers_test + \ + //tensorflow/core:lib_strings_str_util_test + \ + //tensorflow/core/platform/hadoop:hadoop_file_system_test + \ + //tensorflow/core:platform_file_system_test + \ + //tensorflow/core:platform_logging_test + \ + //tensorflow/core:util_sparse_sparse_tensor_test + \ + //tensorflow/cc:framework_gradient_checker_test + \ + //tensorflow/cc:framework_gradients_test + \ + //tensorflow/cc:gradients_array_grad_test + \ + //tensorflow/cc:gradients_math_grad_test + \ + //tensorflow/cc:gradients_nn_grad_test + \ + //tensorflow/cc/saved_model:loader_test +" + +broken_tests="\ + //tensorflow/cc:framework_cc_ops_test + \ + //tensorflow/core/platform/cloud:time_util_test + \ + //tensorflow/core/platform/cloud:oauth_client_test + \ + //tensorflow/core/platform/cloud:http_request_test + \ + //tensorflow/core/platform/cloud:google_auth_provider_test + \ + //tensorflow/core/platform/cloud:gcs_file_system_test + \ + //tensorflow/core/kernels/cloud:bigquery_table_accessor_test + \ + //tensorflow/core/kernels/hexagon:quantized_matmul_op_for_hexagon_test + \ + //tensorflow/core/kernels:sparse_add_op_test + \ + //tensorflow/core/kernels:spacetobatch_benchmark_test_gpu + \ + //tensorflow/core/kernels:spacetobatch_benchmark_test + \ + //tensorflow/core/kernels:requantize_op_test + \ + //tensorflow/core/kernels:requantization_range_op_test + \ + //tensorflow/core/kernels:quantized_reshape_op_test + \ + //tensorflow/core/kernels:quantized_pooling_ops_test + \ + //tensorflow/core/kernels:quantized_matmul_op_test + \ + //tensorflow/core/kernels:quantized_conv_ops_test + \ + //tensorflow/core/kernels:quantized_concat_op_test + \ + //tensorflow/core/kernels:quantized_bias_add_op_test + \ + //tensorflow/core/kernels:quantized_batch_norm_op_test + \ + //tensorflow/core/kernels:quantized_activation_ops_test + \ + //tensorflow/core/kernels:quantize_op_test + \ + //tensorflow/core/kernels:quantize_down_and_shrink_range_op_test + \ + //tensorflow/core/kernels:quantize_and_dequantize_op_test_gpu + \ + //tensorflow/core/kernels:quantize_and_dequantize_op_test + \ + //tensorflow/core/kernels:quantization_utils_test + \ + //tensorflow/core/kernels:debug_ops_test + \ + //tensorflow/core/kernels:control_flow_ops_test + \ + //tensorflow/core/kernels:cast_op_test_gpu + \ + //tensorflow/core/kernels:cast_op_test + \ + //tensorflow/core/distributed_runtime/rpc:rpc_rendezvous_mgr_test_gpu + \ + //tensorflow/core/distributed_runtime/rpc:rpc_rendezvous_mgr_test + \ + //tensorflow/core/distributed_runtime/rpc:grpc_tensor_coding_test + \ + //tensorflow/core/distributed_runtime/rpc:grpc_session_test_gpu + \ + //tensorflow/core/distributed_runtime/rpc:grpc_session_test + \ + //tensorflow/core/distributed_runtime/rpc:grpc_channel_test_gpu + \ + //tensorflow/core/distributed_runtime/rpc:grpc_channel_test + \ + //tensorflow/core/distributed_runtime:remote_device_test_gpu + \ + //tensorflow/core/distributed_runtime:remote_device_test + \ + //tensorflow/core/distributed_runtime:executor_test_gpu + \ + //tensorflow/core/distributed_runtime:executor_test + \ + //tensorflow/core/debug:debug_gateway_test + \ + //tensorflow/core/debug:debug_grpc_io_utils_test + \ + //tensorflow/core:util_reporter_test + \ + //tensorflow/core:util_memmapped_file_system_test + \ + //tensorflow/core:platform_subprocess_test + \ + //tensorflow/core:platform_profile_utils_cpu_utils_test + \ + //tensorflow/core:platform_port_test + \ + //tensorflow/core:lib_strings_strcat_test + \ + //tensorflow/core:lib_jpeg_jpeg_mem_unittest + \ + //tensorflow/core:lib_core_notification_test + \ + //tensorflow/core:framework_partial_tensor_shape_test + \ + //tensorflow/core/debug:debug_io_utils_test \ +" + +exclude_tests="${failing_tests} + ${broken_tests}" + +BUILD_OPTS='-c opt --cpu=x64_windows_msvc --host_cpu=x64_windows_msvc --copt=/w --verbose_failures --experimental_ui' + +# Find all the passing cc_tests on Windows and store them in a variable +passing_tests=$(bazel query "kind(cc_test, //tensorflow/cc/... + //tensorflow/core/...) - (${exclude_tests})" | + # We need to strip \r so that the result could be store into a variable under MSYS + tr '\r' ' ') + +bazel test $BUILD_OPTS -k $passing_tests + diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh index 0e541f3733..28c7475184 100644 --- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh +++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh @@ -32,28 +32,11 @@ set -e # This script is under <repo_root>/tensorflow/tools/ci_build/windows/cpu/pip/ # Change into repository root. script_dir=$(dirname $0) -cd ${script_dir%%tensorflow/tools/ci_build/windows/cpu/pip} +cd ${script_dir%%tensorflow/tools/ci_build/windows/cpu/pip}. -# Use a temporary directory with a short name. -export TMPDIR="C:/tmp" - -# Set bash path -export BAZEL_SH="C:/tools/msys64/usr/bin/bash" - -# Set Python path for ./configure -export PYTHON_BIN_PATH="C:/Program Files/Anaconda3/python" - -# Set Python path for cc_configure.bzl -export BAZEL_PYTHON="C:/Program Files/Anaconda3/python" - -# Set Visual Studio path -export BAZEL_VS="C:/Program Files (x86)/Microsoft Visual Studio 14.0" - -# Add python into PATH, it's needed because gen_git_source.py uses -# '/usr/bin/env python' as a shebang -export PATH="/c/Program Files/Anaconda3:$PATH" - -export TF_NEED_CUDA=0 +# Setting up the environment variables Bazel and ./configure needs +source "tensorflow/tools/ci_build/windows/cpu/bazel/common_env.sh" \ + || { echo "Failed to source common_env.sh" >&2; exit 1; } # bazel clean --expunge doesn't work on Windows yet. # Clean the output base manually to ensure build correctness @@ -65,11 +48,12 @@ bazel shutdown sleep 5 rm -rf ${output_base} +export TF_NEED_CUDA=0 echo "" | ./configure -bazel build -c opt --cpu=x64_windows_msvc --host_cpu=x64_windows_msvc\ - --copt="/w" --verbose_failures --experimental_ui\ - tensorflow/tools/pip_package:build_pip_package || exit $? +BUILD_OPTS='-c opt --cpu=x64_windows_msvc --host_cpu=x64_windows_msvc --copt=/w --verbose_failures --experimental_ui' +bazel build $BUILD_OPTS tensorflow/tools/pip_package:build_pip_package || exit $? ./bazel-bin/tensorflow/tools/pip_package/build_pip_package $PWD + |