Merge pull request #20284 from perfinion:unbundle

PiperOrigin-RevId: 204539752

24 files changed, 557 insertions, 0 deletions

diff --git a/third_party/systemlibs/BUILD b/third_party/systemlibs/BUILD
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/third_party/systemlibs/BUILD
diff --git a/third_party/systemlibs/BUILD.tpl b/third_party/systemlibs/BUILD.tpl
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/third_party/systemlibs/BUILD.tpl
diff --git a/third_party/systemlibs/astor.BUILD b/third_party/systemlibs/astor.BUILD
new file mode 100644
index 0000000000..497ec4bcea
--- /dev/null
+++ b/third_party/systemlibs/astor.BUILD
@@ -0,0 +1,12 @@
+licenses(["notice"])  # New BSD
+
+filegroup(
+    name = "LICENSE",
+    visibility = ["//visibility:public"],
+)
+
+py_library(
+    name = "astor",
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/systemlibs/build_defs.bzl.tpl b/third_party/systemlibs/build_defs.bzl.tpl
new file mode 100644
index 0000000000..3faa46c581
--- /dev/null
+++ b/third_party/systemlibs/build_defs.bzl.tpl
@@ -0,0 +1,32 @@
+# -*- Python -*-
+"""Skylark macros for system libraries.
+"""
+
+SYSTEM_LIBS_ENABLED = %{syslibs_enabled}
+
+SYSTEM_LIBS_LIST = [
+%{syslibs_list}
+]
+
+
+def if_any_system_libs(a, b=[]):
+  """Conditional which evaluates to 'a' if any system libraries are configured."""
+  if SYSTEM_LIBS_ENABLED:
+    return a
+  else:
+    return b
+
+
+def if_system_lib(lib, a, b=[]):
+  """Conditional which evaluates to 'a' if we're using the system version of lib"""
+
+  if SYSTEM_LIBS_ENABLED and lib in SYSTEM_LIBS_LIST:
+    return a
+  else:
+    return b
+
+
+def if_not_system_lib(lib, a, b=[]):
+  """Conditional which evaluates to 'a' if we're using the system version of lib"""
+
+  return if_system_lib(lib, b, a)
diff --git a/third_party/systemlibs/curl.BUILD b/third_party/systemlibs/curl.BUILD
new file mode 100644
index 0000000000..c5f125caa9
--- /dev/null
+++ b/third_party/systemlibs/curl.BUILD
@@ -0,0 +1,12 @@
+licenses(["notice"])  # MIT/X derivative license
+
+filegroup(
+    name = "COPYING",
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "curl",
+    linkopts = ["-lcurl"],
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/systemlibs/cython.BUILD b/third_party/systemlibs/cython.BUILD
new file mode 100644
index 0000000000..1d52587676
--- /dev/null
+++ b/third_party/systemlibs/cython.BUILD
@@ -0,0 +1,13 @@
+licenses(["notice"])  # Apache-2.0
+
+genrule(
+    name = "lncython",
+    outs = ["cython"],
+    cmd = "ln -s $$(which cython) $@",
+)
+
+sh_binary(
+    name = "cython_binary",
+    srcs = ["cython"],
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/systemlibs/flatbuffers.BUILD b/third_party/systemlibs/flatbuffers.BUILD
new file mode 100644
index 0000000000..14fceada82
--- /dev/null
+++ b/third_party/systemlibs/flatbuffers.BUILD
@@ -0,0 +1,38 @@
+licenses(["notice"])  # Apache 2.0
+
+filegroup(
+    name = "LICENSE.txt",
+    visibility = ["//visibility:public"],
+)
+
+# Public flatc library to compile flatbuffer files at runtime.
+cc_library(
+    name = "flatbuffers",
+    linkopts = ["-lflatbuffers"],
+    visibility = ["//visibility:public"],
+)
+
+# Public flatc compiler library.
+cc_library(
+    name = "flatc_library",
+    linkopts = ["-lflatbuffers"],
+    visibility = ["//visibility:public"],
+)
+
+genrule(
+    name = "lnflatc",
+    outs = ["flatc.bin"],
+    cmd = "ln -s $$(which flatc) $@",
+)
+
+# Public flatc compiler.
+sh_binary(
+    name = "flatc",
+    srcs = ["flatc.bin"],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "runtime_cc",
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/systemlibs/gif.BUILD b/third_party/systemlibs/gif.BUILD
new file mode 100644
index 0000000000..5eb2c918ba
--- /dev/null
+++ b/third_party/systemlibs/gif.BUILD
@@ -0,0 +1,12 @@
+licenses(["notice"])  # MIT
+
+filegroup(
+    name = "COPYING",
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "gif",
+    linkopts = ["-lgif"],
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/systemlibs/grpc.BUILD b/third_party/systemlibs/grpc.BUILD
new file mode 100644
index 0000000000..fd90eb0dd3
--- /dev/null
+++ b/third_party/systemlibs/grpc.BUILD
@@ -0,0 +1,54 @@
+licenses(["notice"])  # Apache v2
+
+filegroup(
+    name = "LICENSE",
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "grpc",
+    linkopts = ["-lgrpc"],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "grpc++",
+    linkopts = ["-lgrpc++"],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "grpc_unsecure",
+    linkopts = ["-lgrpc_unsecure"],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "grpc++_unsecure",
+    linkopts = ["-lgrpc++_unsecure"],
+    visibility = ["//visibility:public"],
+)
+
+genrule(
+    name = "ln_grpc_cpp_plugin",
+    outs = ["grpc_cpp_plugin.bin"],
+    cmd = "ln -s $$(which grpc_cpp_plugin) $@",
+)
+
+sh_binary(
+    name = "grpc_cpp_plugin",
+    srcs = ["grpc_cpp_plugin.bin"],
+    visibility = ["//visibility:public"],
+)
+
+genrule(
+    name = "ln_grpc_python_plugin",
+    outs = ["grpc_python_plugin.bin"],
+    cmd = "ln -s $$(which grpc_python_plugin) $@",
+)
+
+sh_binary(
+    name = "grpc_python_plugin",
+    srcs = ["grpc_python_plugin.bin"],
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/systemlibs/jemalloc.BUILD b/third_party/systemlibs/jemalloc.BUILD
new file mode 100644
index 0000000000..6a48d582ba
--- /dev/null
+++ b/third_party/systemlibs/jemalloc.BUILD
@@ -0,0 +1,30 @@
+licenses(["notice"])  # BSD
+
+filegroup(
+    name = "COPYING",
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "jemalloc_headers",
+    defines = [
+        "jemalloc_posix_memalign=posix_memalign",
+        "jemalloc_malloc=malloc",
+        "jemalloc_realloc=realloc",
+        "jemalloc_free=free",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "jemalloc_impl",
+    linkopts = ["-ljemalloc"],
+    defines = [
+        "jemalloc_posix_memalign=posix_memalign",
+        "jemalloc_malloc=malloc",
+        "jemalloc_realloc=realloc",
+        "jemalloc_free=free",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":jemalloc_headers"],
+)
diff --git a/third_party/systemlibs/jpeg.BUILD b/third_party/systemlibs/jpeg.BUILD
new file mode 100644
index 0000000000..f4f52da9bd
--- /dev/null
+++ b/third_party/systemlibs/jpeg.BUILD
@@ -0,0 +1,12 @@
+licenses(["notice"])  # custom notice-style license, see LICENSE.md
+
+filegroup(
+    name = "LICENSE.md",
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "jpeg",
+    linkopts = ["-ljpeg"],
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/systemlibs/jsoncpp.BUILD b/third_party/systemlibs/jsoncpp.BUILD
new file mode 100644
index 0000000000..cf91917cfb
--- /dev/null
+++ b/third_party/systemlibs/jsoncpp.BUILD
@@ -0,0 +1,37 @@
+licenses(["unencumbered"])  # Public Domain or MIT
+
+filegroup(
+    name = "LICENSE",
+    visibility = ["//visibility:public"],
+)
+
+HEADERS = [
+    "include/json/autolink.h",
+    "include/json/config.h",
+    "include/json/features.h",
+    "include/json/forwards.h",
+    "include/json/json.h",
+    "include/json/reader.h",
+    "include/json/value.h",
+    "include/json/version.h",
+    "include/json/writer.h",
+]
+
+genrule(
+    name = "link_headers",
+    outs = HEADERS,
+    cmd = """
+      for i in $(OUTS); do
+        i=$${i##*/}
+        ln -vsf /usr/include/jsoncpp/json/$$i $(@D)/include/json/$$i
+      done
+    """,
+)
+
+cc_library(
+    name = "jsoncpp",
+    hdrs = HEADERS,
+    includes = ["."],
+    linkopts = ["-ljsoncpp"],
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/systemlibs/lmdb.BUILD b/third_party/systemlibs/lmdb.BUILD
new file mode 100644
index 0000000000..6177b095ec
--- /dev/null
+++ b/third_party/systemlibs/lmdb.BUILD
@@ -0,0 +1,12 @@
+licenses(["notice"])  # OpenLDAP Public License
+
+filegroup(
+    name = "LICENSE",
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "lmdb",
+    linkopts = ["-llmdb"],
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/systemlibs/nasm.BUILD b/third_party/systemlibs/nasm.BUILD
new file mode 100644
index 0000000000..10ef8d8832
--- /dev/null
+++ b/third_party/systemlibs/nasm.BUILD
@@ -0,0 +1,12 @@
+licenses(["notice"])  # BSD 2-clause
+
+filegroup(
+    name = "LICENSE",
+    visibility = ["//visibility:public"],
+)
+
+sh_binary(
+    name = "nasm",
+    srcs = ["nasm"],
+    visibility = ["@jpeg//:__pkg__"],
+)
diff --git a/third_party/systemlibs/pcre.BUILD b/third_party/systemlibs/pcre.BUILD
new file mode 100644
index 0000000000..df74238847
--- /dev/null
+++ b/third_party/systemlibs/pcre.BUILD
@@ -0,0 +1,12 @@
+licenses(["notice"])  # BSD
+
+filegroup(
+    name = "LICENCE",
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "pcre",
+    linkopts = ["-lpcre"],
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/systemlibs/png.BUILD b/third_party/systemlibs/png.BUILD
new file mode 100644
index 0000000000..fc6b6f2d8b
--- /dev/null
+++ b/third_party/systemlibs/png.BUILD
@@ -0,0 +1,12 @@
+licenses(["notice"])  # BSD/MIT-like license
+
+filegroup(
+    name = "LICENSE",
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "png",
+    linkopts = ["-lpng"],
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/systemlibs/re2.BUILD b/third_party/systemlibs/re2.BUILD
new file mode 100644
index 0000000000..c18e252dbc
--- /dev/null
+++ b/third_party/systemlibs/re2.BUILD
@@ -0,0 +1,12 @@
+licenses(["notice"])  # BSD/MIT-like license
+
+filegroup(
+    name = "LICENSE",
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "re2",
+    linkopts = ["-lre2"],
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/systemlibs/six.BUILD b/third_party/systemlibs/six.BUILD
new file mode 100644
index 0000000000..ff9b1a540b
--- /dev/null
+++ b/third_party/systemlibs/six.BUILD
@@ -0,0 +1,11 @@
+licenses(["notice"])  # MIT
+
+filegroup(
+    name = "LICENSE",
+    visibility = ["//visibility:public"],
+)
+
+py_library(
+    name = "six",
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/systemlibs/snappy.BUILD b/third_party/systemlibs/snappy.BUILD
new file mode 100644
index 0000000000..fd2db9e2df
--- /dev/null
+++ b/third_party/systemlibs/snappy.BUILD
@@ -0,0 +1,12 @@
+licenses(["notice"])  # BSD 3-Clause
+
+filegroup(
+    name = "COPYING",
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "snappy",
+    linkopts = ["-lsnappy"],
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/systemlibs/sqlite.BUILD b/third_party/systemlibs/sqlite.BUILD
new file mode 100644
index 0000000000..20ee1ebbef
--- /dev/null
+++ b/third_party/systemlibs/sqlite.BUILD
@@ -0,0 +1,15 @@
+licenses(["unencumbered"])  # Public Domain
+
+# Production build of SQLite library that's baked into TensorFlow.
+cc_library(
+    name = "org_sqlite",
+    linkopts = ["-lsqlite3"],
+    visibility = ["//visibility:public"],
+)
+
+# This is a Copybara sync helper for Google.
+py_library(
+    name = "python",
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/systemlibs/swig.BUILD b/third_party/systemlibs/swig.BUILD
new file mode 100644
index 0000000000..4c9b74dadb
--- /dev/null
+++ b/third_party/systemlibs/swig.BUILD
@@ -0,0 +1,23 @@
+licenses(["restricted"])  # GPLv3
+
+filegroup(
+    name = "LICENSE",
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "templates",
+    visibility = ["//visibility:public"],
+)
+
+genrule(
+    name = "lnswiglink",
+    outs = ["swiglink"],
+    cmd = "ln -s $$(which swig) $@",
+)
+
+sh_binary(
+    name = "swig",
+    srcs = ["swiglink"],
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/systemlibs/syslibs_configure.bzl b/third_party/systemlibs/syslibs_configure.bzl
new file mode 100644
index 0000000000..07a44c317e
--- /dev/null
+++ b/third_party/systemlibs/syslibs_configure.bzl
@@ -0,0 +1,160 @@
+# -*- Python -*-
+"""Repository rule for system library autoconfiguration.
+
+`syslibs_configure` depends on the following environment variables:
+
+  * `TF_SYSTEM_LIBS`: list of third party dependencies that should use
+    the system version instead
+"""
+
+_TF_SYSTEM_LIBS="TF_SYSTEM_LIBS"
+
+VALID_LIBS=[
+    "astor_archive",
+    "com_googlesource_code_re2",
+    "curl",
+    "cython",
+    "flatbuffers",
+    "gif_archive",
+    "grpc",
+    "jemalloc",
+    "jpeg",
+    "jsoncpp_git",
+    "lmdb",
+    "nasm",
+    "org_sqlite",
+    "pcre",
+    "png_archive",
+    "six_archive",
+    "snappy",
+    "swig",
+    "termcolor_archive",
+    "zlib_archive",
+]
+
+
+def auto_configure_fail(msg):
+  """Output failure message when syslibs configuration fails."""
+  red = "\033[0;31m"
+  no_color = "\033[0m"
+  fail("\n%sSystem Library Configuration Error:%s %s\n" % (red, no_color, msg))
+
+
+def _is_windows(repository_ctx):
+  """Returns true if the host operating system is windows."""
+  os_name = repository_ctx.os.name.lower()
+  if os_name.find("windows") != -1:
+    return True
+  return False
+
+
+def _enable_syslibs(repository_ctx):
+  s = repository_ctx.os.environ.get(_TF_SYSTEM_LIBS, '').strip()
+  if not _is_windows(repository_ctx) and s != None and s != '':
+    return True
+  return False
+
+
+def _get_system_lib_list(repository_ctx):
+  """Gets the list of deps that should use the system lib.
+
+  Args:
+    repository_ctx: The repository context.
+
+  Returns:
+    A string version of a python list
+  """
+  if _TF_SYSTEM_LIBS not in repository_ctx.os.environ:
+    return []
+
+  libenv = repository_ctx.os.environ[_TF_SYSTEM_LIBS].strip()
+  libs = []
+
+  for lib in list(libenv.split(',')):
+    lib = lib.strip()
+    if lib == "":
+      continue
+    if lib not in VALID_LIBS:
+      auto_configure_fail("Invalid system lib set: %s" % lib)
+      return []
+    libs.append(lib)
+
+  return libs
+
+
+def _format_system_lib_list(repository_ctx):
+  """Formats the list of deps that should use the system lib.
+
+  Args:
+    repository_ctx: The repository context.
+
+  Returns:
+    A list of the names of deps that should use the system lib.
+  """
+  libs = _get_system_lib_list(repository_ctx)
+  ret = ''
+  for lib in libs:
+    ret += "'%s',\n" % lib
+
+  return ret
+
+
+def _tpl(repository_ctx, tpl, substitutions={}, out=None):
+  if not out:
+    out = tpl.replace(":", "")
+  repository_ctx.template(
+      out,
+      Label("//third_party/systemlibs%s.tpl" % tpl),
+      substitutions,
+      False)
+
+
+def _create_dummy_repository(repository_ctx):
+  """Creates the dummy repository to build with all bundled libraries."""
+
+  _tpl(repository_ctx, ":BUILD")
+  _tpl(repository_ctx, ":build_defs.bzl",
+    {
+      "%{syslibs_enabled}": 'False',
+      "%{syslibs_list}": '',
+    })
+
+
+def _create_local_repository(repository_ctx):
+  """Creates the repository to build with system libraries."""
+
+  _tpl(repository_ctx, ":BUILD")
+  _tpl(repository_ctx, ":build_defs.bzl",
+    {
+      "%{syslibs_enabled}": 'True',
+      "%{syslibs_list}": _format_system_lib_list(repository_ctx),
+    })
+
+
+def _syslibs_autoconf_impl(repository_ctx):
+  """Implementation of the syslibs_configure repository rule."""
+  if not _enable_syslibs(repository_ctx):
+    _create_dummy_repository(repository_ctx)
+  else:
+    _create_local_repository(repository_ctx)
+
+
+syslibs_configure = repository_rule(
+    implementation = _syslibs_autoconf_impl,
+    environ = [
+        _TF_SYSTEM_LIBS,
+    ],
+)
+
+"""Configures the build to link to system libraries
+instead of using bundled versions.
+
+Add the following to your WORKSPACE FILE:
+
+```python
+syslibs_configure(name = "local_config_syslibs")
+```
+
+Args:
+  name: A unique name for this workspace rule.
+"""
diff --git a/third_party/systemlibs/termcolor.BUILD b/third_party/systemlibs/termcolor.BUILD
new file mode 100644
index 0000000000..915eb621d5
--- /dev/null
+++ b/third_party/systemlibs/termcolor.BUILD
@@ -0,0 +1,12 @@
+licenses(["notice"])  # MIT
+
+filegroup(
+    name = "COPYING.txt",
+    visibility = ["//visibility:public"],
+)
+
+py_library(
+    name = "termcolor",
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/systemlibs/zlib.BUILD b/third_party/systemlibs/zlib.BUILD
new file mode 100644
index 0000000000..69462ae6cb
--- /dev/null
+++ b/third_party/systemlibs/zlib.BUILD
@@ -0,0 +1,12 @@
+licenses(["notice"])  # BSD/MIT-like license (for zlib)
+
+filegroup(
+    name = "zlib.h",
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "zlib",
+    linkopts = ["-lz"],
+    visibility = ["//visibility:public"],
+)