From efbdc15b280374607895ab0ada467de4a0512e0c Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Fri, 1 Dec 2017 16:02:57 -0800 Subject: Introduce tf_http_archive (#15018) This new repository rule consolidates patched_http_archive, temp_workaround_http_archive, http_archive, and new_http_archive. The following behaviors have been introduced: - A delete attribute that can rm -rf certain repo content after extraction - Helpful error messages when mirroring requirements aren't followed --- third_party/aws.BUILD | 16 +++---- third_party/curl.BUILD | 38 ++++++++-------- third_party/gif.BUILD | 2 +- third_party/jemalloc.BUILD | 10 ++--- third_party/jpeg/jpeg.BUILD | 2 +- third_party/nccl.BUILD | 8 ++-- third_party/repo.bzl | 103 ++++++++++++++++++++++++++++++++++++++++++++ third_party/snappy.BUILD | 4 +- 8 files changed, 143 insertions(+), 40 deletions(-) create mode 100644 third_party/repo.bzl (limited to 'third_party') diff --git a/third_party/aws.BUILD b/third_party/aws.BUILD index bc9e37ffb3..bf5310aa16 100644 --- a/third_party/aws.BUILD +++ b/third_party/aws.BUILD @@ -7,21 +7,21 @@ licenses(["notice"]) # Apache 2.0 exports_files(["LICENSE"]) -load("@%ws%//third_party:common.bzl", "template_rule") +load("@org_tensorflow//third_party:common.bzl", "template_rule") cc_library( name = "aws", srcs = select({ - "@%ws%//tensorflow:linux_x86_64": glob([ + "@org_tensorflow//tensorflow:linux_x86_64": glob([ "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp", ]), - "@%ws%//tensorflow:darwin": glob([ + "@org_tensorflow//tensorflow:darwin": glob([ "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp", ]), - "@%ws%//tensorflow:linux_ppc64le": glob([ + "@org_tensorflow//tensorflow:linux_ppc64le": glob([ "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp", ]), - "@%ws%//tensorflow:raspberry_pi_armeabi": glob([ + "@org_tensorflow//tensorflow:raspberry_pi_armeabi": glob([ "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp", ]), "//conditions:default": [], @@ -53,17 +53,17 @@ cc_library( "aws-cpp-sdk-core/include/aws/core/SDKConfig.h", ], defines = select({ - "@%ws%//tensorflow:linux_x86_64": [ + "@org_tensorflow//tensorflow:linux_x86_64": [ "PLATFORM_LINUX", "ENABLE_CURL_CLIENT", "ENABLE_NO_ENCRYPTION", ], - "@%ws%//tensorflow:darwin": [ + "@org_tensorflow//tensorflow:darwin": [ "PLATFORM_APPLE", "ENABLE_CURL_CLIENT", "ENABLE_NO_ENCRYPTION", ], - "@%ws%//tensorflow:linux_ppc64le": [ + "@org_tensorflow//tensorflow:linux_ppc64le": [ "PLATFORM_LINUX", "ENABLE_CURL_CLIENT", "ENABLE_NO_ENCRYPTION", diff --git a/third_party/curl.BUILD b/third_party/curl.BUILD index 805a30d262..0f6c75a210 100644 --- a/third_party/curl.BUILD +++ b/third_party/curl.BUILD @@ -224,14 +224,14 @@ cc_library( "lib/wildcard.h", "lib/x509asn1.h", ] + select({ - "@%ws%//tensorflow:darwin": [ + "@org_tensorflow//tensorflow:darwin": [ "lib/vtls/darwinssl.c", ], - "@%ws%//tensorflow:ios": [ + "@org_tensorflow//tensorflow:ios": [ "lib/vtls/darwinssl.c", ], - "@%ws%//tensorflow:windows": CURL_WIN_SRCS, - "@%ws%//tensorflow:windows_msvc": CURL_WIN_SRCS, + "@org_tensorflow//tensorflow:windows": CURL_WIN_SRCS, + "@org_tensorflow//tensorflow:windows_msvc": CURL_WIN_SRCS, "//conditions:default": [ "lib/vtls/openssl.c", ], @@ -248,8 +248,8 @@ cc_library( "include/curl/typecheck-gcc.h", ], copts = select({ - "@%ws%//tensorflow:windows": CURL_WIN_COPTS, - "@%ws%//tensorflow:windows_msvc": CURL_WIN_COPTS, + "@org_tensorflow//tensorflow:windows": CURL_WIN_COPTS, + "@org_tensorflow//tensorflow:windows_msvc": CURL_WIN_COPTS, "//conditions:default": [ "-I%prefix%/curl/lib", "-D_GNU_SOURCE", @@ -261,14 +261,14 @@ cc_library( "-Wno-string-plus-int", ], }) + select({ - "@%ws%//tensorflow:darwin": [ + "@org_tensorflow//tensorflow:darwin": [ "-fno-constant-cfstrings", ], - "@%ws%//tensorflow:windows": [ + "@org_tensorflow//tensorflow:windows": [ # See curl.h for discussion of write size and Windows "/DCURL_MAX_WRITE_SIZE=16384", ], - "@%ws%//tensorflow:windows_msvc": [ + "@org_tensorflow//tensorflow:windows_msvc": [ # See curl.h for discussion of write size and Windows "/DCURL_MAX_WRITE_SIZE=16384", ], @@ -278,20 +278,20 @@ cc_library( }), includes = ["include"], linkopts = select({ - "@%ws%//tensorflow:android": [ + "@org_tensorflow//tensorflow:android": [ "-pie", ], - "@%ws%//tensorflow:darwin": [ + "@org_tensorflow//tensorflow:darwin": [ "-Wl,-framework", "-Wl,CoreFoundation", "-Wl,-framework", "-Wl,Security", ], - "@%ws%//tensorflow:ios": [], - "@%ws%//tensorflow:windows": [ + "@org_tensorflow//tensorflow:ios": [], + "@org_tensorflow//tensorflow:windows": [ "-Wl,ws2_32.lib", ], - "@%ws%//tensorflow:windows_msvc": [ + "@org_tensorflow//tensorflow:windows_msvc": [ "-Wl,ws2_32.lib", ], "//conditions:default": [ @@ -302,9 +302,9 @@ cc_library( deps = [ "@zlib_archive//:zlib", ] + select({ - "@%ws%//tensorflow:ios": [], - "@%ws%//tensorflow:windows": [], - "@%ws%//tensorflow:windows_msvc": [], + "@org_tensorflow//tensorflow:ios": [], + "@org_tensorflow//tensorflow:windows": [], + "@org_tensorflow//tensorflow:windows_msvc": [], "//conditions:default": [ "@boringssl//:ssl", ], @@ -406,8 +406,8 @@ cc_binary( "src/tool_xattr.h", ], copts = select({ - "@%ws%//tensorflow:windows": CURL_BIN_WIN_COPTS, - "@%ws%//tensorflow:windows_msvc": CURL_BIN_WIN_COPTS, + "@org_tensorflow//tensorflow:windows": CURL_BIN_WIN_COPTS, + "@org_tensorflow//tensorflow:windows_msvc": CURL_BIN_WIN_COPTS, "//conditions:default": [ "-I%prefix%/curl/lib", "-D_GNU_SOURCE", diff --git a/third_party/gif.BUILD b/third_party/gif.BUILD index 27808a9d64..78fbd6c0e0 100644 --- a/third_party/gif.BUILD +++ b/third_party/gif.BUILD @@ -21,7 +21,7 @@ cc_library( ], hdrs = ["lib/gif_lib.h"], defines = select({ - #"@%ws%//tensorflow:android": [ + #"@org_tensorflow//tensorflow:android": [ ":android": [ "S_IREAD=S_IRUSR", "S_IWRITE=S_IWUSR", diff --git a/third_party/jemalloc.BUILD b/third_party/jemalloc.BUILD index a2addf2c66..1b0829b8fe 100644 --- a/third_party/jemalloc.BUILD +++ b/third_party/jemalloc.BUILD @@ -5,7 +5,7 @@ licenses(["notice"]) # BSD exports_files(["COPYING"]) -load("@%ws%//third_party:common.bzl", "template_rule") +load("@org_tensorflow//third_party:common.bzl", "template_rule") cc_library( name = "jemalloc_headers", @@ -97,10 +97,10 @@ cc_library( includes = ["include"], # pthread_atfork() is called for PPC. linkopts = select({ - "@%ws%//tensorflow:linux_ppc64le": [ + "@org_tensorflow//tensorflow:linux_ppc64le": [ "-lpthread", ], - "@%ws%//tensorflow:linux_x86_64": [ + "@org_tensorflow//tensorflow:linux_x86_64": [ "-lpthread", ], "//conditions:default": [ @@ -208,8 +208,8 @@ genrule( name = "size_classes_h", outs = ["include/jemalloc/internal/size_classes.h"], cmd = select({ - "@%ws%//tensorflow:linux_ppc64le": "$(location :size_classes_sh) \"3 4\" 3 16 2 >$@", - "@%ws%//tensorflow:linux_x86_64": "$(location :size_classes_sh) \"3 4\" 3 12 2 >$@", + "@org_tensorflow//tensorflow:linux_ppc64le": "$(location :size_classes_sh) \"3 4\" 3 16 2 >$@", + "@org_tensorflow//tensorflow:linux_x86_64": "$(location :size_classes_sh) \"3 4\" 3 12 2 >$@", "//conditions:default": "$(location :size_classes_sh) \"3 4\" 3 12 2 >$@", }), tools = [":size_classes_sh"], diff --git a/third_party/jpeg/jpeg.BUILD b/third_party/jpeg/jpeg.BUILD index f6078052ec..e431f19382 100644 --- a/third_party/jpeg/jpeg.BUILD +++ b/third_party/jpeg/jpeg.BUILD @@ -5,7 +5,7 @@ licenses(["notice"]) # custom notice-style license, see LICENSE.md exports_files(["LICENSE.md"]) -load("@%ws%//third_party:common.bzl", "template_rule") +load("@org_tensorflow//third_party:common.bzl", "template_rule") libjpegturbo_nocopts = "-[W]error" diff --git a/third_party/nccl.BUILD b/third_party/nccl.BUILD index 8c7b9bdbe9..b2b8e18824 100644 --- a/third_party/nccl.BUILD +++ b/third_party/nccl.BUILD @@ -44,17 +44,17 @@ cc_library( "-O3", ] + cuda_default_copts(), linkopts = select({ - "@%ws%//tensorflow:android": [ + "@org_tensorflow//tensorflow:android": [ "-pie", ], - "@%ws%//tensorflow:darwin": [ + "@org_tensorflow//tensorflow:darwin": [ "-Wl,-framework", "-Wl,CoreFoundation", "-Wl,-framework", "-Wl,Security", ], - "@%ws%//tensorflow:ios": [], - "@%ws%//tensorflow:windows": [ + "@org_tensorflow//tensorflow:ios": [], + "@org_tensorflow//tensorflow:windows": [ "-DEFAULTLIB:ws2_32.lib", ], "//conditions:default": [ diff --git a/third_party/repo.bzl b/third_party/repo.bzl new file mode 100644 index 0000000000..d6e5dfced0 --- /dev/null +++ b/third_party/repo.bzl @@ -0,0 +1,103 @@ +# Copyright 2017 The TensorFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utilities for defining TensorFlow Bazel dependencies.""" + +_SINGLE_URL_WHITELIST = depset([ + "arm_compiler", + "ortools_archive", +]) + +def _is_windows(ctx): + return ctx.os.name.lower().find("windows") != -1 + +def _get_env_var(ctx, name): + if name in ctx.os.environ: + return ctx.os.environ[name] + else: + return None + +# Executes specified command with arguments and calls 'fail' if it exited with +# non-zero code +def _execute_and_check_ret_code(repo_ctx, cmd_and_args): + result = repo_ctx.execute(cmd_and_args, timeout=10) + if result.return_code != 0: + fail(("Non-zero return code({1}) when executing '{0}':\n" + "Stdout: {2}\n" + + "Stderr: {3}").format(" ".join(cmd_and_args), result.return_code, + result.stdout, result.stderr)) + +def _repos_are_siblings(): + return Label("@foo//bar").workspace_root.startswith("../") + +# Apply a patch_file to the repository root directory +# Runs 'patch -p1' +def _apply_patch(ctx, patch_file): + # Don't check patch on Windows, because patch is only available under bash. + if not _is_windows(ctx) and not ctx.which("patch"): + fail("patch command is not found, please install it") + cmd = ["patch", "-p1", "-d", ctx.path("."), "-i", ctx.path(patch_file)] + if _is_windows(ctx): + bazel_sh = _get_env_var(ctx, "BAZEL_SH") + if not bazel_sh: + fail("BAZEL_SH environment variable is not set") + cmd = [bazel_sh, "-c", " ".join(cmd)] + _execute_and_check_ret_code(ctx, cmd) + +def _apply_delete(ctx, paths): + for path in paths: + if path.startswith("/"): + fail("refusing to rm -rf path starting with '/': " + path) + if ".." in path: + fail("refusing to rm -rf path containing '..': " + path) + _execute_and_check_ret_code( + ctx, ["rm", "-rf"] + [ctx.path(path) for path in paths]) + +def _tf_http_archive(ctx): + if ("mirror.bazel.build" not in ctx.attr.urls[0] or + (len(ctx.attr.urls) < 2 and + ctx.attr.name not in _SINGLE_URL_WHITELIST)): + fail("tf_http_archive(urls) must have redundant URLs. The Bazel Mirror " + + "URL must come first. Please note mirroring happens after merge") + ctx.download_and_extract( + ctx.attr.urls, + "", + ctx.attr.sha256, + ctx.attr.type, + ctx.attr.strip_prefix) + if ctx.attr.delete: + _apply_delete(ctx, ctx.attr.delete) + if ctx.attr.patch_file != None: + _apply_patch(ctx, ctx.attr.patch_file) + if ctx.attr.build_file != None: + ctx.template("BUILD", ctx.attr.build_file, { + "%prefix%": ".." if _repos_are_siblings() else "external", + }, False) + +tf_http_archive = repository_rule( + implementation=_tf_http_archive, + attrs={ + "sha256": attr.string(mandatory=True), + "urls": attr.string_list(mandatory=True, allow_empty=False), + "strip_prefix": attr.string(), + "type": attr.string(), + "delete": attr.string_list(), + "patch_file": attr.label(), + "build_file": attr.label(), + }) +"""Downloads and creates Bazel repos for dependencies. + +This is a swappable replacement for both http_archive() and +new_http_archive() that offers some additional features. It also helps +ensure best practices are followed. +""" diff --git a/third_party/snappy.BUILD b/third_party/snappy.BUILD index 9c00b7068a..fd48ed8941 100644 --- a/third_party/snappy.BUILD +++ b/third_party/snappy.BUILD @@ -50,8 +50,8 @@ genrule( "-e 's/@ac_cv_have_stddef_h@/1/g' " + "-e 's/@ac_cv_have_stdint_h@/1/g' " + select({ - "@%ws%//tensorflow:windows": "-e 's/@ac_cv_have_sys_uio_h@/0/g' ", - "@%ws%//tensorflow:windows_msvc": "-e 's/@ac_cv_have_sys_uio_h@/0/g' ", + "@org_tensorflow//tensorflow:windows": "-e 's/@ac_cv_have_sys_uio_h@/0/g' ", + "@org_tensorflow//tensorflow:windows_msvc": "-e 's/@ac_cv_have_sys_uio_h@/0/g' ", "//conditions:default": "-e 's/@ac_cv_have_sys_uio_h@/1/g' ", }) + "-e 's/@SNAPPY_MAJOR@/1/g' " + -- cgit v1.2.3