diff options
| -rw-r--r-- | third_party/gpus/crosstool/CROSSTOOL.tpl | 249 | ||||
| -rw-r--r-- | third_party/gpus/cuda/platform.bzl.tpl | 15 | ||||
| -rw-r--r-- | third_party/nccl/BUILD | 0 | ||||
| -rw-r--r-- | third_party/nccl/fix_clang_compilation.patch | 85 | ||||
| -rw-r--r-- | third_party/nccl/nccl.BUILD | 66 | ||||
| -rw-r--r-- | tools/bazel.rc.template | 39 | ||||
| -rwxr-xr-x | util/python/python_config.sh | 159 |
7 files changed, 0 insertions, 613 deletions
diff --git a/third_party/gpus/crosstool/CROSSTOOL.tpl b/third_party/gpus/crosstool/CROSSTOOL.tpl deleted file mode 100644 index b77a45c325..0000000000 --- a/third_party/gpus/crosstool/CROSSTOOL.tpl +++ /dev/null @@ -1,249 +0,0 @@ -major_version: "local" -minor_version: "" -default_target_cpu: "same_as_host" - -default_toolchain { - cpu: "k8" - toolchain_identifier: "local_linux" -} -default_toolchain { - cpu: "piii" - toolchain_identifier: "local_linux" -} -default_toolchain { - cpu: "arm" - toolchain_identifier: "local_linux" -} -default_toolchain { - cpu: "darwin" - toolchain_identifier: "local_darwin" -} -default_toolchain { - cpu: "ppc" - toolchain_identifier: "local_linux" -} - -toolchain { - abi_version: "local" - abi_libc_version: "local" - builtin_sysroot: "" - compiler: "compiler" - host_system_name: "local" - needsPic: true - supports_gold_linker: false - supports_incremental_linker: false - supports_fission: false - supports_interface_shared_objects: false - supports_normalizing_ar: false - supports_start_end_lib: false - supports_thin_archives: false - target_libc: "local" - target_cpu: "local" - target_system_name: "local" - toolchain_identifier: "local_linux" - - tool_path { name: "ar" path: "/usr/bin/ar" } - tool_path { name: "compat-ld" path: "/usr/bin/ld" } - tool_path { name: "cpp" path: "/usr/bin/cpp" } - tool_path { name: "dwp" path: "/usr/bin/dwp" } - # As part of the TensorFlow release, we place some cuda-related compilation - # files in @local_config_cuda//crosstool/clang/bin, and this relative - # path, combined with the rest of our Bazel configuration causes our - # compilation to use those files. - tool_path { name: "gcc" path: "clang/bin/crosstool_wrapper_driver_is_not_gcc" } - # Use "-std=c++11" for nvcc. For consistency, force both the host compiler - # and the device compiler to use "-std=c++11". - cxx_flag: "-std=c++11" - linker_flag: "-Wl,-no-as-needed" - linker_flag: "-lstdc++" - linker_flag: "-B/usr/bin/" - -%{gcc_host_compiler_includes} - tool_path { name: "gcov" path: "/usr/bin/gcov" } - - # C(++) compiles invoke the compiler (as that is the one knowing where - # to find libraries), but we provide LD so other rules can invoke the linker. - tool_path { name: "ld" path: "/usr/bin/ld" } - - tool_path { name: "nm" path: "/usr/bin/nm" } - tool_path { name: "objcopy" path: "/usr/bin/objcopy" } - objcopy_embed_flag: "-I" - objcopy_embed_flag: "binary" - tool_path { name: "objdump" path: "/usr/bin/objdump" } - tool_path { name: "strip" path: "/usr/bin/strip" } - - # Anticipated future default. - unfiltered_cxx_flag: "-no-canonical-prefixes" - - # Make C++ compilation deterministic. Use linkstamping instead of these - # compiler symbols. - unfiltered_cxx_flag: "-Wno-builtin-macro-redefined" - unfiltered_cxx_flag: "-D__DATE__=\"redacted\"" - unfiltered_cxx_flag: "-D__TIMESTAMP__=\"redacted\"" - unfiltered_cxx_flag: "-D__TIME__=\"redacted\"" - - # Security hardening on by default. - # Conservative choice; -D_FORTIFY_SOURCE=2 may be unsafe in some cases. - # We need to undef it before redefining it as some distributions now have - # it enabled by default. - compiler_flag: "-U_FORTIFY_SOURCE" - compiler_flag: "-D_FORTIFY_SOURCE=1" - compiler_flag: "-fstack-protector" - compiler_flag: "-fPIE" - linker_flag: "-pie" - linker_flag: "-Wl,-z,relro,-z,now" - - # Enable coloring even if there's no attached terminal. Bazel removes the - # escape sequences if --nocolor is specified. This isn't supported by gcc - # on Ubuntu 14.04. - # compiler_flag: "-fcolor-diagnostics" - - # All warnings are enabled. Maybe enable -Werror as well? - compiler_flag: "-Wall" - # Enable a few more warnings that aren't part of -Wall. - compiler_flag: "-Wunused-but-set-parameter" - # But disable some that are problematic. - compiler_flag: "-Wno-free-nonheap-object" # has false positives - - # Keep stack frames for debugging, even in opt mode. - compiler_flag: "-fno-omit-frame-pointer" - - # Anticipated future default. - linker_flag: "-no-canonical-prefixes" - unfiltered_cxx_flag: "-fno-canonical-system-headers" - # Have gcc return the exit code from ld. - linker_flag: "-pass-exit-codes" - # Stamp the binary with a unique identifier. - linker_flag: "-Wl,--build-id=md5" - linker_flag: "-Wl,--hash-style=gnu" - # Gold linker only? Can we enable this by default? - # linker_flag: "-Wl,--warn-execstack" - # linker_flag: "-Wl,--detect-odr-violations" - - # Include directory for cuda headers. - cxx_builtin_include_directory: "%{cuda_include_path}" - - compilation_mode_flags { - mode: DBG - # Enable debug symbols. - compiler_flag: "-g" - } - compilation_mode_flags { - mode: OPT - - # No debug symbols. - # Maybe we should enable https://gcc.gnu.org/wiki/DebugFission for opt or - # even generally? However, that can't happen here, as it requires special - # handling in Bazel. - compiler_flag: "-g0" - - # Conservative choice for -O - # -O3 can increase binary size and even slow down the resulting binaries. - # Profile first and / or use FDO if you need better performance than this. - compiler_flag: "-O2" - - # Disable assertions - compiler_flag: "-DNDEBUG" - - # Removal of unused code and data at link time (can this increase binary size in some cases?). - compiler_flag: "-ffunction-sections" - compiler_flag: "-fdata-sections" - linker_flag: "-Wl,--gc-sections" - } - linking_mode_flags { mode: DYNAMIC } -} - -toolchain { - abi_version: "local" - abi_libc_version: "local" - builtin_sysroot: "" - compiler: "compiler" - host_system_name: "local" - needsPic: true - target_libc: "macosx" - target_cpu: "darwin" - target_system_name: "local" - toolchain_identifier: "local_darwin" - - tool_path { name: "ar" path: "/usr/bin/libtool" } - tool_path { name: "compat-ld" path: "/usr/bin/ld" } - tool_path { name: "cpp" path: "/usr/bin/cpp" } - tool_path { name: "dwp" path: "/usr/bin/dwp" } - tool_path { name: "gcc" path: "clang/bin/crosstool_wrapper_driver_is_not_gcc" } - cxx_flag: "-std=c++11" - ar_flag: "-static" - ar_flag: "-s" - ar_flag: "-o" - linker_flag: "-lc++" - linker_flag: "-undefined" - linker_flag: "dynamic_lookup" - # TODO(ulfjack): This is wrong on so many levels. Figure out a way to auto-detect the proper - # setting from the local compiler, and also how to make incremental builds correct. - cxx_builtin_include_directory: "/" - tool_path { name: "gcov" path: "/usr/bin/gcov" } - tool_path { name: "ld" path: "/usr/bin/ld" } - tool_path { name: "nm" path: "/usr/bin/nm" } - tool_path { name: "objcopy" path: "/usr/bin/objcopy" } - objcopy_embed_flag: "-I" - objcopy_embed_flag: "binary" - tool_path { name: "objdump" path: "/usr/bin/objdump" } - tool_path { name: "strip" path: "/usr/bin/strip" } - - # Anticipated future default. - unfiltered_cxx_flag: "-no-canonical-prefixes" - # Make C++ compilation deterministic. Use linkstamping instead of these - # compiler symbols. - unfiltered_cxx_flag: "-Wno-builtin-macro-redefined" - unfiltered_cxx_flag: "-D__DATE__=\"redacted\"" - unfiltered_cxx_flag: "-D__TIMESTAMP__=\"redacted\"" - unfiltered_cxx_flag: "-D__TIME__=\"redacted\"" - - # Security hardening on by default. - # Conservative choice; -D_FORTIFY_SOURCE=2 may be unsafe in some cases. - compiler_flag: "-D_FORTIFY_SOURCE=1" - compiler_flag: "-fstack-protector" - - # Enable coloring even if there's no attached terminal. Bazel removes the - # escape sequences if --nocolor is specified. - compiler_flag: "-fcolor-diagnostics" - - # All warnings are enabled. Maybe enable -Werror as well? - compiler_flag: "-Wall" - # Enable a few more warnings that aren't part of -Wall. - compiler_flag: "-Wthread-safety" - compiler_flag: "-Wself-assign" - - # Keep stack frames for debugging, even in opt mode. - compiler_flag: "-fno-omit-frame-pointer" - - # Anticipated future default. - linker_flag: "-no-canonical-prefixes" - - # Include directory for cuda headers. - cxx_builtin_include_directory: "%{cuda_include_path}" - - compilation_mode_flags { - mode: DBG - # Enable debug symbols. - compiler_flag: "-g" - } - compilation_mode_flags { - mode: OPT - # No debug symbols. - # Maybe we should enable https://gcc.gnu.org/wiki/DebugFission for opt or even generally? - # However, that can't happen here, as it requires special handling in Bazel. - compiler_flag: "-g0" - - # Conservative choice for -O - # -O3 can increase binary size and even slow down the resulting binaries. - # Profile first and / or use FDO if you need better performance than this. - compiler_flag: "-O2" - - # Disable assertions - compiler_flag: "-DNDEBUG" - - # Removal of unused code and data at link time (can this increase binary size in some cases?). - compiler_flag: "-ffunction-sections" - compiler_flag: "-fdata-sections" - } -} diff --git a/third_party/gpus/cuda/platform.bzl.tpl b/third_party/gpus/cuda/platform.bzl.tpl deleted file mode 100644 index 01ef24b94e..0000000000 --- a/third_party/gpus/cuda/platform.bzl.tpl +++ /dev/null @@ -1,15 +0,0 @@ -CUDA_VERSION = "%{cuda_version}" -CUDNN_VERSION = "%{cudnn_version}" -PLATFORM = "%{platform}" - -def cuda_sdk_version(): - return CUDA_VERSION - -def cudnn_sdk_version(): - return CUDNN_VERSION - -def readlink_command(): - if PLATFORM == "Darwin": - return "greadlink" - else: - return "readlink" diff --git a/third_party/nccl/BUILD b/third_party/nccl/BUILD deleted file mode 100644 index e69de29bb2..0000000000 --- a/third_party/nccl/BUILD +++ /dev/null diff --git a/third_party/nccl/fix_clang_compilation.patch b/third_party/nccl/fix_clang_compilation.patch deleted file mode 100644 index e8d2a7dc9f..0000000000 --- a/third_party/nccl/fix_clang_compilation.patch +++ /dev/null @@ -1,85 +0,0 @@ -From 8241cd7b6ed1425eeb88fd380090575978e358f4 Mon Sep 17 00:00:00 2001 -From: Ilya Biryukov <ibiryukov@google.com> -Date: Thu, 16 Mar 2017 12:01:11 +0100 -Subject: [PATCH 1/1] Fix compilation error when compiling with 'clang -x - cuda'. - -Functions vFetch and vStore are not found by ADL with clang, -so they need to be declared before usage in ReduceCopy. ---- - src/common_kernel.h | 52 ++++++++++++++++++++++++++-------------------------- - 1 file changed, 26 insertions(+), 26 deletions(-) - -diff --git a/src/common_kernel.h b/src/common_kernel.h -index 28fbc85..cc71f8a 100644 ---- a/src/common_kernel.h -+++ b/src/common_kernel.h -@@ -30,6 +30,32 @@ - #define BAR(type, barid, nthreads) \ - BAR_EXPAND(type, barid, ROUNDUP(nthreads, WARP_SIZE)) - -+template<typename T> inline __device__ -+T vFetch(const volatile T* ptr) { -+ return *ptr; -+} -+ -+#ifdef CUDA_HAS_HALF -+template<> inline __device__ -+half vFetch<half>(const volatile half* ptr) { -+ half r; -+ r.x = ptr->x; -+ return r; -+} -+#endif -+ -+template<typename T> inline __device__ -+void vStore(volatile T* ptr, const T val) { -+ *ptr = val; -+} -+ -+#ifdef CUDA_HAS_HALF -+template<> inline __device__ -+void vStore<half>(volatile half* ptr, const half val) { -+ ptr->x = val.x; -+} -+#endif -+ - __device__ unsigned int spinct; - - // Spin wait until func evaluates to true -@@ -225,32 +251,6 @@ __device__ inline volatile T* AlignUp(volatile T * ptr, size_t align) { - return reinterpret_cast<volatile T*>(ALIGNUP(ptrval, align)); - } - --template<typename T> inline __device__ --T vFetch(const volatile T* ptr) { -- return *ptr; --} -- --#ifdef CUDA_HAS_HALF --template<> inline __device__ --half vFetch<half>(const volatile half* ptr) { -- half r; -- r.x = ptr->x; -- return r; --} --#endif -- --template<typename T> inline __device__ --void vStore(volatile T* ptr, const T val) { -- *ptr = val; --} -- --#ifdef CUDA_HAS_HALF --template<> inline __device__ --void vStore<half>(volatile half* ptr, const half val) { -- ptr->x = val.x; --} --#endif -- - // Assumptions: - // - there is exactly 1 block - // - THREADS is the number of producer threads --- -2.12.0.367.g23dc2f6d3c-goog - diff --git a/third_party/nccl/nccl.BUILD b/third_party/nccl/nccl.BUILD deleted file mode 100644 index 06b9b8ff68..0000000000 --- a/third_party/nccl/nccl.BUILD +++ /dev/null @@ -1,66 +0,0 @@ -# NVIDIA nccl -# A package of optimized primitives for collective multi-GPU communication. - -licenses(["notice"]) # BSD - -exports_files(["LICENSE.txt"]) - -load("@local_config_cuda//cuda:build_defs.bzl", "cuda_default_copts", "if_cuda") - -SRCS = [ - "src/all_gather.cu", - "src/all_reduce.cu", - "src/broadcast.cu", - "src/core.cu", - "src/libwrap.cu", - "src/reduce.cu", - "src/reduce_scatter.cu", -] - -# Copy .cu to .cu.cc so they can be in srcs of cc_library. -[ - genrule( - name = "gen_" + src, - srcs = [src], - outs = [src + ".cc"], - cmd = "cp $(location " + src + ") $(location " + src + ".cc)", - ) - for src in SRCS -] - -SRCS_CU_CC = [src + ".cc" for src in SRCS] - -cc_library( - name = "nccl", - srcs = if_cuda(SRCS_CU_CC + glob(["src/*.h"])), - hdrs = if_cuda(["src/nccl.h"]), - copts = [ - "-DCUDA_MAJOR=0", - "-DCUDA_MINOR=0", - "-DNCCL_MAJOR=0", - "-DNCCL_MINOR=0", - "-DNCCL_PATCH=0", - "-Iexternal/nccl_archive/src", - "-O3", - ] + cuda_default_copts(), - linkopts = select({ - "@%ws%//tensorflow:android": [ - "-pie", - ], - "@%ws%//tensorflow:darwin": [ - "-Wl,-framework", - "-Wl,CoreFoundation", - "-Wl,-framework", - "-Wl,Security", - ], - "@%ws%//tensorflow:ios": [], - "@%ws%//tensorflow:windows": [ - "ws2_32.lib", - ], - "//conditions:default": [ - "-lrt", - ], - }), - visibility = ["//visibility:public"], - deps = ["@local_config_cuda//cuda:cuda_headers"], -) diff --git a/tools/bazel.rc.template b/tools/bazel.rc.template deleted file mode 100644 index 097ff7b9d0..0000000000 --- a/tools/bazel.rc.template +++ /dev/null @@ -1,39 +0,0 @@ -build:cuda --crosstool_top=@local_config_cuda//crosstool:toolchain -build:cuda --define=using_cuda=true --define=using_cuda_nvcc=true - -build:cuda_clang --crosstool_top=@local_config_cuda//crosstool:toolchain -build:cuda_clang --define=using_cuda=true --define=using_cuda_clang=true - -build:win-cuda --define=using_cuda=true --define=using_cuda_nvcc=true - -build:mkl --define=using_mkl=true - -build:sycl --crosstool_top=@local_config_sycl//crosstool:toolchain -build:sycl --define=using_sycl=true - -build:sycl_asan --crosstool_top=@local_config_sycl//crosstool:toolchain -build:sycl_asan --define=using_sycl=true --copt -fno-omit-frame-pointer --copt -fsanitize-coverage=3 --copt -DGPR_NO_DIRECT_SYSCALLS --linkopt -fPIC --linkopt -fsanitize=address - -build --force_python=py$PYTHON_MAJOR_VERSION -build --host_force_python=py$PYTHON_MAJOR_VERSION -build --python$PYTHON_MAJOR_VERSION_path=$PYTHON_BINARY -build --define=use_fast_cpp_protos=true -build --define=allow_oversize_protos=true - -build --define PYTHON_BIN_PATH=$PYTHON_BINARY -test --define PYTHON_BIN_PATH=$PYTHON_BINARY -test --force_python=py$PYTHON_MAJOR_VERSION -test --host_force_python=py$PYTHON_MAJOR_VERSION -run --define PYTHON_BIN_PATH=$PYTHON_BINARY - -build --spawn_strategy=standalone -test --spawn_strategy=standalone -run --spawn_strategy=standalone - -build --genrule_strategy=standalone -test --genrule_strategy=standalone -run --genrule_strategy=standalone - -build -c opt -test -c opt -run -c opt diff --git a/util/python/python_config.sh b/util/python/python_config.sh deleted file mode 100755 index d5762ad456..0000000000 --- a/util/python/python_config.sh +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -set -e -o errexit - -if [ -d "../org_tensorflow" ]; then - script_path="../org_tensorflow" -else - # Prefix expected paths with ./ locally and external/reponame/ for remote repos. - # TODO(kchodorow): remove once runfiles paths are fixed, see - # https://github.com/bazelbuild/bazel/issues/848. - script_path=$(dirname $(dirname $(dirname "$0"))) - script_path=${script_path:-.} -fi - -function main { - setup_python "$1" - exit 0 -} - -function python_path { - "$PYTHON_BIN_PATH" - <<END -from __future__ import print_function -import site -import os - -try: - input = raw_input -except NameError: - pass - -python_paths = [] -if os.getenv('PYTHONPATH') is not None: - python_paths = os.getenv('PYTHONPATH').split(':') -try: - library_paths = site.getsitepackages() -except AttributeError: - from distutils.sysconfig import get_python_lib - library_paths = [get_python_lib()] -all_paths = set(python_paths + library_paths) - -paths = [] -for path in all_paths: - if os.path.isdir(path): - paths.append(path) - -if len(paths) == 1: - print(paths[0]) -else: - ret_paths = ",".join(paths) - print(ret_paths) -END -} - -function default_python_path { - PYTHON_ARG="$1" "$PYTHON_BIN_PATH" - <<END -from __future__ import print_function -import os - -default = os.getenv('PYTHON_ARG') -default = str(default) -print(default) -END -} - -function setup_python { - PYTHON_BIN_PATH="$1"; - - # TODO(ngiraldo): move most of these checks to root configure - if [ -z "$PYTHON_BIN_PATH" ]; then - echo "PYTHON_BIN_PATH was not provided. Did you run configure?" - exit 1 - fi - if [ ! -x "$PYTHON_BIN_PATH" ] || [ -d "$PYTHON_BIN_PATH" ]; then - echo "PYTHON_BIN_PATH is not executable. Is it the python binary?" - exit 1 - fi - - local python_major_version=$("${PYTHON_BIN_PATH}" -c 'from __future__ import print_function; import sys; print(sys.version_info[0]);') - if [ "$python_major_version" == "" ]; then - echo -e "\n\nERROR: Problem getting python version. Is $PYTHON_BIN_PATH the correct python binary?" - exit 1 - fi - - # TODO(ngiraldo): confirm if these checks are really necessary, remove if not - if [ -z "$PYTHON_LIB_PATH" ]; then - local python_lib_path - # Split python_path into an array of paths, this allows path containing spaces - IFS=',' - python_lib_path=($(python_path)) - unset IFS - - if [ 1 = "$USE_DEFAULT_PYTHON_LIB_PATH" ]; then - PYTHON_LIB_PATH="$(default_python_path "${python_lib_path[0]}")" - echo "Using python library path: $PYTHON_LIB_PATH" - - else - echo "Found possible Python library paths:" - for x in "${python_lib_path[@]}"; do - echo " $x" - done - set -- "${python_lib_path[@]}" - echo "Please input the desired Python library path to use. Default is ["$1"]" - read b || true - if [ "$b" == "" ]; then - PYTHON_LIB_PATH="$(default_python_path "${python_lib_path[0]}")" - echo "Using python library path: $PYTHON_LIB_PATH" - else - PYTHON_LIB_PATH="$b" - fi - fi - fi - - if test -d "$PYTHON_LIB_PATH" -a -x "$PYTHON_LIB_PATH"; then - python_lib="$PYTHON_LIB_PATH" - else - echo -e "\n\nERROR: Invalid python library path: ${PYTHON_LIB_PATH}." - exit 1 - fi - - # Convert python path to Windows style before writing into bazel.rc - if is_windows; then - PYTHON_BIN_PATH="$(cygpath -m "$PYTHON_BIN_PATH")" - fi - - # TODO(ngiraldo): move all below to root configure - # Write tools/bazel.rc - echo "# Autogenerated by configure: DO NOT EDIT" > tools/bazel.rc - sed -e "s/\$PYTHON_MAJOR_VERSION/$python_major_version/g" \ - -e "s|\$PYTHON_BINARY|\"$PYTHON_BIN_PATH\"|g" \ - tools/bazel.rc.template >> tools/bazel.rc - # Write tools/python_bin_path.sh - echo "export PYTHON_BIN_PATH=\"$PYTHON_BIN_PATH\"" > tools/python_bin_path.sh -} - -PLATFORM="$(uname -s | tr 'A-Z' 'a-z')" -function is_windows() { - # On windows, the shell script is actually running in msys - if [[ "${PLATFORM}" =~ msys_nt* ]]; then - true - else - false - fi -} - -main "$@" |