diff options
author | 2018-03-15 15:44:56 -0700 | |
---|---|---|
committer | 2018-03-15 15:49:01 -0700 | |
commit | 1e75c69339da2fbf2c5c5fbeb891243badae7ff8 (patch) | |
tree | 81268169bbff6836bfbbd4e9866a1374f597a624 /tensorflow/tools/def_file_filter | |
parent | 6c62e650252ab32f83637a8de6720e73ffeca226 (diff) |
Automated g4 rollback of changelist 189231636
PiperOrigin-RevId: 189258641
Diffstat (limited to 'tensorflow/tools/def_file_filter')
-rw-r--r-- | tensorflow/tools/def_file_filter/BUILD | 0 | ||||
-rw-r--r-- | tensorflow/tools/def_file_filter/BUILD.tpl | 15 | ||||
-rw-r--r-- | tensorflow/tools/def_file_filter/def_file_filter.py.tpl | 168 | ||||
-rw-r--r-- | tensorflow/tools/def_file_filter/def_file_filter_configure.bzl | 56 |
4 files changed, 0 insertions, 239 deletions
diff --git a/tensorflow/tools/def_file_filter/BUILD b/tensorflow/tools/def_file_filter/BUILD deleted file mode 100644 index e69de29bb2..0000000000 --- a/tensorflow/tools/def_file_filter/BUILD +++ /dev/null diff --git a/tensorflow/tools/def_file_filter/BUILD.tpl b/tensorflow/tools/def_file_filter/BUILD.tpl deleted file mode 100644 index 3cb72f4979..0000000000 --- a/tensorflow/tools/def_file_filter/BUILD.tpl +++ /dev/null @@ -1,15 +0,0 @@ -# Description: -# Tools for filtering DEF file for TensorFlow on Windows -# -# On Windows, we use a DEF file generated by Bazel to export -# symbols from the tensorflow dynamic library(_pywrap_tensorflow.dll). -# The maximum number of symbols that can be exported per DLL is 64K, -# so we have to filter some useless symbols through this python script. - -package(default_visibility = ["//visibility:public"]) - -py_binary( - name = "def_file_filter", - srcs = ["def_file_filter.py"], - srcs_version = "PY2AND3", -) diff --git a/tensorflow/tools/def_file_filter/def_file_filter.py.tpl b/tensorflow/tools/def_file_filter/def_file_filter.py.tpl deleted file mode 100644 index 8bdc03eb0f..0000000000 --- a/tensorflow/tools/def_file_filter/def_file_filter.py.tpl +++ /dev/null @@ -1,168 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""def_file_filter.py - tool to filter a windows def file. - -The def file can be used to export symbols from the tensorflow dll to enable -tf.load_library(). - -Because the linker allows only 64K symbols to be exported per dll -we filter the symbols down to the essentials. The regular expressions -we use for this are specific to tensorflow. - -TODO: this works fine but there is an issue with exporting -'const char * const' and importing it from a user_ops. The problem is -on the importing end and using __declspec(dllimport) works around it. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import io -import os -import re -import subprocess -import sys -import tempfile - -# External tools we use that come with visual studio sdk -UNDNAME = "%{undname_bin_path}" - -# Exclude if matched -EXCLUDE_RE = re.compile(r"RTTI|deleting destructor|::internal::") - -# Include if matched before exclude -INCLUDEPRE_RE = re.compile(r"google::protobuf::internal::ExplicitlyConstructed|" - r"google::protobuf::internal::ArenaImpl::AllocateAligned|" # for contrib/data/_prefetching_ops - r"google::protobuf::internal::ArenaImpl::AddCleanup|" # for contrib/data/_prefetching_ops - r"google::protobuf::Arena::OnArenaAllocation|" # for contrib/data/_prefetching_ops - r"tensorflow::internal::LogMessage|" - r"tensorflow::internal::LogString|" - r"tensorflow::internal::CheckOpMessageBuilder|" - r"tensorflow::internal::MakeCheckOpValueString|" - r"tensorflow::internal::PickUnusedPortOrDie|" - r"tensorflow::internal::ValidateDevice|" - r"tensorflow::ops::internal::Enter|" - r"tensorflow::strings::internal::AppendPieces|" - r"tensorflow::strings::internal::CatPieces|" - r"tensorflow::io::internal::JoinPathImpl") - -# Include if matched after exclude -INCLUDE_RE = re.compile(r"^(TF_\w*)$|" - r"^(TFE_\w*)$|" - r"nsync::|" - r"tensorflow::|" - r"functor::|" - r"perftools::gputools") - -# We want to identify data members explicitly in the DEF file, so that no one -# can implicitly link against the DLL if they use one of the variables exported -# from the DLL and the header they use does not decorate the symbol with -# __declspec(dllimport). It is easier to detect what a data symbol does -# NOT look like, so doing it with the below regex. -DATA_EXCLUDE_RE = re.compile(r"[)(]|" - r"vftable|" - r"vbtable|" - r"vcall|" - r"RTTI|" - r"protobuf::internal::ExplicitlyConstructed") - -def get_args(): - """Parse command line.""" - filename_list = lambda x: x.split(";") - parser = argparse.ArgumentParser() - parser.add_argument("--input", type=filename_list, - help="paths to input def file", - required=True) - parser.add_argument("--output", help="output deffile", required=True) - parser.add_argument("--target", help="name of the target", required=True) - args = parser.parse_args() - return args - - -def main(): - """main.""" - args = get_args() - - # Pipe dumpbin to extract all linkable symbols from libs. - # Good symbols are collected in candidates and also written to - # a temp file. - candidates = [] - tmpfile = tempfile.NamedTemporaryFile(mode="w", delete=False) - for def_file_path in args.input: - def_file = open(def_file_path, 'r') - for line in def_file: - cols = line.split() - sym = cols[0] - tmpfile.file.write(sym + "\n") - candidates.append(sym) - tmpfile.file.close() - - # Run the symbols through undname to get their undecorated name - # so we can filter on something readable. - with open(args.output, "w") as def_fp: - # track dupes - taken = set() - - # Header for the def file. - def_fp.write("LIBRARY " + args.target + "\n") - def_fp.write("EXPORTS\n") - def_fp.write("\t ??1OpDef@tensorflow@@UEAA@XZ\n") - - # Each symbols returned by undname matches the same position in candidates. - # We compare on undname but use the decorated name from candidates. - dupes = 0 - proc = subprocess.Popen([UNDNAME, tmpfile.name], stdout=subprocess.PIPE) - for idx, line in enumerate(io.TextIOWrapper(proc.stdout, encoding="utf-8")): - decorated = candidates[idx] - if decorated in taken: - # Symbol is already in output, done. - dupes += 1 - continue - - if not INCLUDEPRE_RE.search(line): - if EXCLUDE_RE.search(line): - continue - if not INCLUDE_RE.search(line): - continue - - if "deleting destructor" in line: - # Some of the symbols convered by INCLUDEPRE_RE export deleting - # destructor symbols, which is a bad idea. - # So we filter out such symbols here. - continue - - if DATA_EXCLUDE_RE.search(line): - def_fp.write("\t" + decorated + "\n") - else: - def_fp.write("\t" + decorated + " DATA\n") - taken.add(decorated) - def_fp.close() - - exit_code = proc.wait() - if exit_code != 0: - print("{} failed, exit={}".format(UNDNAME, exit_code)) - return exit_code - - os.unlink(tmpfile.name) - - print("symbols={}, taken={}, dupes={}" - .format(len(candidates), len(taken), dupes)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tensorflow/tools/def_file_filter/def_file_filter_configure.bzl b/tensorflow/tools/def_file_filter/def_file_filter_configure.bzl deleted file mode 100644 index 47539b2423..0000000000 --- a/tensorflow/tools/def_file_filter/def_file_filter_configure.bzl +++ /dev/null @@ -1,56 +0,0 @@ -"""Repository rule for def file filter autoconfiguration. - -This repository reuses Bazel's VC detect mechanism to find undname.exe, -which is a tool used in def_file_filter.py. - -def_file_filter.py is for filtering the DEF file for TensorFlow on Windows. -On Windows, we use a DEF file generated by Bazel to export symbols from the -tensorflow dynamic library(_pywrap_tensorflow.dll). The maximum number of -symbols that can be exported per DLL is 64K, so we have to filter some useless -symbols through this python script. - -`def_file_filter_config` depends on the following environment variables: - * `BAZEL_VC` - * `BAZEL_VS` - * `VS90COMNTOOLS` - * `VS100COMNTOOLS` - * `VS110COMNTOOLS` - * `VS120COMNTOOLS` - * `VS140COMNTOOLS` -""" - -load("@bazel_tools//tools/cpp:windows_cc_configure.bzl", "find_vc_path") -load("@bazel_tools//tools/cpp:windows_cc_configure.bzl", "find_msvc_tool") -load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "auto_configure_fail") - -def _def_file_filter_configure_impl(repository_ctx): - if repository_ctx.os.name.lower().find("windows") == -1: - repository_ctx.symlink(Label("//tensorflow/tools/def_file_filter:BUILD.tpl"), "BUILD") - repository_ctx.file("def_file_filter.py", "") - return - vc_path = find_vc_path(repository_ctx) - if vc_path == "visual-studio-not-found": - auto_configure_fail("Visual C++ build tools not found on your machine") - undname_bin_path = find_msvc_tool(repository_ctx, vc_path, "undname.exe").replace("\\", "\\\\") - - repository_ctx.template( - "def_file_filter.py", - Label("//tensorflow/tools/def_file_filter:def_file_filter.py.tpl"), - { - "%{undname_bin_path}": undname_bin_path, - }) - repository_ctx.symlink(Label("//tensorflow/tools/def_file_filter:BUILD.tpl"), "BUILD") - - -def_file_filter_configure = repository_rule( - implementation = _def_file_filter_configure_impl, - environ = [ - "BAZEL_VC", - "BAZEL_VS", - "VS90COMNTOOLS", - "VS100COMNTOOLS", - "VS110COMNTOOLS", - "VS120COMNTOOLS", - "VS140COMNTOOLS" - ], -) |