diff options
| author |  Benoit Steiner <bsteiner@google.com> | 2017-05-10 21:12:21 -0700 |
|---|---|---|
| committer |  TensorFlower Gardener <gardener@tensorflow.org> | 2017-05-11 11:02:28 -0700 |
| commit | ee112cff56081fb9d0b74c987a8935acc360b05c (patch) | |
| tree | 6026d8b42ccc09d9c0d1b2d091916cfcb4f5a057 | |
| parent | 27c89207d2f31fe4b4b42c789b96d62cde4e2133 (diff) | |
Merge changes from github.
PiperOrigin-RevId: 155709893
125 files changed, 2306 insertions, 687 deletions
diff --git a/.gitignore b/.gitignore index da5d14a9b7..bdcb067fc2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ node_modules /.bazelrc /.tf_configure.bazelrc /bazel-* +/bazel_pip +/third_party/eigen3/mkl_include +/third_party/mkl/* /tools/python_bin_path.sh /tools/git/gen /pip_test diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c36ef1ecd3..c78b6b1a15 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,7 +40,7 @@ TensorFlow coding style. * Include unit tests when you contribute new features, as they help to a) prove that your code works correctly, b) guard against future breaking changes to lower the maintenance cost. -* Bug fixes also generally require unit tests, because the presense of bugs +* Bug fixes also generally require unit tests, because the presence of bugs usually indicates insufficient test coverage. * Keep API compatibility in mind when you change code in core TensorFlow, e.g., code in [tensorflow/core](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/core) and [tensorflow/python](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/python). @@ -147,17 +147,17 @@ function setup_python { # Set-up env variables used by python_configure.bzl write_action_env_to_bazelrc "PYTHON_BIN_PATH" "$PYTHON_BIN_PATH" write_action_env_to_bazelrc "PYTHON_LIB_PATH" "$PYTHON_LIB_PATH" - write_to_bazelrc "build --define PYTHON_BIN_PATH=$PYTHON_BIN_PATH" - write_to_bazelrc "build --define PYTHON_LIB_PATH=$PYTHON_LIB_PATH" + write_to_bazelrc "build --define PYTHON_BIN_PATH=\"$PYTHON_BIN_PATH\"" + write_to_bazelrc "build --define PYTHON_LIB_PATH=\"$PYTHON_LIB_PATH\"" write_to_bazelrc "build --force_python=py$python_major_version" write_to_bazelrc "build --host_force_python=py$python_major_version" - write_to_bazelrc "build --python${python_major_version}_path=$PYTHON_BIN_PATH" + write_to_bazelrc "build --python${python_major_version}_path=\"$PYTHON_BIN_PATH\"" write_to_bazelrc "test --force_python=py$python_major_version" write_to_bazelrc "test --host_force_python=py$python_major_version" - write_to_bazelrc "test --define PYTHON_BIN_PATH=$PYTHON_BIN_PATH" - write_to_bazelrc "test --define PYTHON_LIB_PATH=$PYTHON_LIB_PATH" - write_to_bazelrc "run --define PYTHON_BIN_PATH=$PYTHON_BIN_PATH" - write_to_bazelrc "run --define PYTHON_LIB_PATH=$PYTHON_LIB_PATH" + write_to_bazelrc "test --define PYTHON_BIN_PATH=\"$PYTHON_BIN_PATH\"" + write_to_bazelrc "test --define PYTHON_LIB_PATH=\"$PYTHON_LIB_PATH\"" + write_to_bazelrc "run --define PYTHON_BIN_PATH=\"$PYTHON_BIN_PATH\"" + write_to_bazelrc "run --define PYTHON_LIB_PATH=\"$PYTHON_LIB_PATH\"" # Write tools/python_bin_path.sh echo "export PYTHON_BIN_PATH=\"$PYTHON_BIN_PATH\"" > tools/python_bin_path.sh @@ -180,35 +180,60 @@ fi setup_python ## Set up MKL related environment settings -if false; then # Disable building with MKL for now - while [ "$TF_NEED_MKL" == "" ]; do +while [ "$TF_NEED_MKL" == "" ]; do + fromuser="" + read -p "Do you wish to build TensorFlow with MKL support? [y/N] " INPUT + fromuser="1" + case $INPUT in + [Yy]* ) echo "MKL support will be enabled for TensorFlow"; TF_NEED_MKL=1;; + [Nn]* ) echo "No MKL support will be enabled for TensorFlow"; TF_NEED_MKL=0;; + "" ) echo "No MKL support will be enabled for TensorFlow"; TF_NEED_MKL=0;; + * ) echo "Invalid selection: " $INPUT;; + esac +done + +OSNAME=`uname -s` + +if [ "$TF_NEED_MKL" == "1" ]; then # TF_NEED_MKL + while [ "$TF_DOWNLOAD_MKL" == "" ]; do fromuser="" - read -p "Do you wish to build TensorFlow with MKL support (experimental)? [y/N] " INPUT + read -p "Do you wish to download MKL LIB from the web? [Y/n] " INPUT fromuser="1" case $INPUT in - [Yy]* ) echo "MKL support (experimental) (will be enabled for TensorFlow"; TF_NEED_MKL=1;; - [Nn]* ) echo "No MKL support will be enabled for TensorFlow"; TF_NEED_MKL=0;; - "" ) echo "No MKL support will be enabled for TensorFlow"; TF_NEED_MKL=0;; - * ) echo "Invalid selection: " $INPUT;; + [Yy]* ) TF_DOWNLOAD_MKL=1;; + [Nn]* ) TF_DOWNLOAD_MKL=0;; + "" ) TF_DOWNLOAD_MKL=1;; + * ) echo "Invalid selection: " $INPUT; exit 1;; esac done - OSNAME=`uname -s` - - if [ "$TF_NEED_MKL" == "1" ]; then # TF_NEED_MKL + if [[ "$TF_DOWNLOAD_MKL" == "1" ]]; then DST=`dirname $0` - ARCHIVE_BASENAME=mklml_lnx_2017.0.2.20170209.tgz - GITHUB_RELEASE_TAG=v0.5 + ARCHIVE_BASENAME=mklml_lnx_2018.0.20170425.tgz + GITHUB_RELEASE_TAG=v0.7 MKLURL="https://github.com/01org/mkl-dnn/releases/download/$GITHUB_RELEASE_TAG/$ARCHIVE_BASENAME" - if ! [ -e "$DST/third_party/mkl/$ARCHIVE_BASENAME" ]; then - wget --no-check-certificate -P $DST/third_party/mkl/ $MKLURL + if ! [ -e "${DST}/third_party/mkl/${ARCHIVE_BASENAME}" ]; then + curl -fSsL -o "${DST}/third_party/mkl/${ARCHIVE_BASENAME}" "${MKLURL}" fi tar -xzf $DST/third_party/mkl/$ARCHIVE_BASENAME -C $DST/third_party/mkl/ extracted_dir_name="${ARCHIVE_BASENAME%.*}" MKL_INSTALL_PATH=$DST/third_party/mkl/$extracted_dir_name MKL_INSTALL_PATH=`${PYTHON_BIN_PATH} -c "import os; print(os.path.realpath(os.path.expanduser('${MKL_INSTALL_PATH}')))"` - if [ "$OSNAME" == "Linux" ]; then + else + default_mkl_path=/opt/intel/mklml + fromuser="" + read -p "Please specify the location where MKL is installed. [Default is $default_mkl_path]: " MKL_INSTALL_PATH + fromuser="1" + if [ -z "$MKL_INSTALL_PATH" ]; then + MKL_INSTALL_PATH=$default_mkl_path + fi + # Result returned from "read" will be used unexpanded. That make "~" unuseable. + # Going through one more level of expansion to handle that. + MKL_INSTALL_PATH=`${PYTHON_BIN_PATH} -c "import os; print(os.path.realpath(os.path.expanduser('${MKL_INSTALL_PATH}')))"` + fi + + if [ "$OSNAME" == "Linux" ]; then # Full MKL configuration MKL_RT_LIB_PATH="lib/intel64/libmkl_rt.so" #${TF_MKL_EXT}#TODO version? MKL_RT_OMP_LIB_PATH="../compiler/lib/intel64/libiomp5.so" #TODO VERSION? @@ -216,24 +241,29 @@ if false; then # Disable building with MKL for now # MKL-ML configuration MKL_ML_LIB_PATH="lib/libmklml_intel.so" #${TF_MKL_EXT}#TODO version? MKL_ML_OMP_LIB_PATH="lib/libiomp5.so" #TODO VERSION? - elif [ "$OSNAME" == "Darwin" ]; then + elif [ "$OSNAME" == "Darwin" ]; then echo "Darwin is unsupported yet"; exit 1 - fi + fi - if [ -e "$MKL_INSTALL_PATH/${MKL_ML_LIB_PATH}" ]; then + if [ -e "$MKL_INSTALL_PATH/${MKL_ML_LIB_PATH}" ]; then ln -sf $MKL_INSTALL_PATH/${MKL_ML_LIB_PATH} third_party/mkl/ ln -sf $MKL_INSTALL_PATH/${MKL_ML_OMP_LIB_PATH} third_party/mkl/ ln -sf $MKL_INSTALL_PATH/include third_party/mkl/ ln -sf $MKL_INSTALL_PATH/include third_party/eigen3/mkl_include - else - echo "ERROR: $MKL_INSTALL_PATH/${MKL_ML_LIB_PATH} does not exist"; - exit 1 - fi - - if [ -z "$fromuser" ]; then + loc=$(locate -e libdl.so.2 | sed -n 1p) + ln -sf $loc third_party/mkl/libdl.so.2 + elif [ -e "$MKL_INSTALL_PATH/${MKL_RT_LIB_PATH}" ]; then + ln -sf $MKL_INSTALL_PATH/${MKL_RT_LIB_PATH} third_party/mkl/ + ln -sf $MKL_INSTALL_PATH/${MKL_RT_OMP_LIB_PATH} third_party/mkl/ + ln -sf $MKL_INSTALL_PATH/include third_party/mkl/ + ln -sf $MKL_INSTALL_PATH/include third_party/eigen3/mkl_include + loc=$(locate -e libdl.so.2 | sed -n 1p) + ln -sf $loc third_party/mkl/libdl.so.2 + else + echo "ERROR: $MKL_INSTALL_PATH/${MKL_ML_LIB_PATH} nor $MKL_INSTALL_PATH/${MKL_RT_LIB_PATH} exists"; exit 1 - fi + fi cat > third_party/mkl/mkl.config <<EOF # MKL_INSTALL_PATH refers to the location of MKL root folder. The MKL header and library @@ -241,9 +271,8 @@ cat > third_party/mkl/mkl.config <<EOF MKL_INSTALL_PATH=$MKL_INSTALL_PATH EOF - fi # TF_NEED_MKL - ################## MKL -fi # Disable building with MKL for now +fi # TF_NEED_MKL +## End MKL setup ## Set up architecture-dependent optimization flags. if [ -z "$CC_OPT_FLAGS" ]; then diff --git a/tensorflow/compiler/xla/service/allocation_tracker.cc b/tensorflow/compiler/xla/service/allocation_tracker.cc index 49a621810e..83759a7a0c 100644 --- a/tensorflow/compiler/xla/service/allocation_tracker.cc +++ b/tensorflow/compiler/xla/service/allocation_tracker.cc @@ -64,8 +64,9 @@ GlobalDataHandle AllocationTracker::RegisterInternal( auto& allocation = FindOrDie(handle_to_allocation_, handle); int ref_count = allocation->ref_count(); CHECK_GT(ref_count, 0); - VLOG(2) << "ref_count: " << ref_count << " -> " << ref_count + 1; - allocation->increment_ref_count(); + VLOG(2) << "ref_count: " << ref_count << " -> " << + (ref_count + initial_ref_count); + allocation->increment_ref_count(initial_ref_count); } else { handle = next_handle_++; VLOG(2) << "ref_count: " << initial_ref_count; diff --git a/tensorflow/compiler/xla/service/allocation_tracker.h b/tensorflow/compiler/xla/service/allocation_tracker.h index e007680016..ebbf35b6fe 100644 --- a/tensorflow/compiler/xla/service/allocation_tracker.h +++ b/tensorflow/compiler/xla/service/allocation_tracker.h @@ -63,10 +63,10 @@ class Allocation { CHECK_GE(ref_count_, 0); return ref_count_; } - void increment_ref_count() { + void increment_ref_count(int inc) { CHECK_GT(ref_count_, 0); - CHECK_LT(ref_count_, INT_MAX); - ++ref_count_; + CHECK_LE(ref_count_, INT_MAX - inc); + ref_count_ += inc; } void decrement_ref_count() { CHECK_GT(ref_count_, 0); diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD index fbfad70098..affb5f9906 100644 --- a/tensorflow/compiler/xla/service/cpu/BUILD +++ b/tensorflow/compiler/xla/service/cpu/BUILD @@ -97,6 +97,7 @@ cc_library( name = "simple_orc_jit", srcs = ["simple_orc_jit.cc"], hdrs = ["simple_orc_jit.h"], + linkopts = ["-ldl"], deps = [ ":compiler_functor", ":cpu_runtime", diff --git a/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_stream.h b/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_stream.h index daf0e48000..fd577ad712 100644 --- a/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_stream.h +++ b/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_stream.h @@ -17,6 +17,7 @@ #include <memory> #include <vector> +#include <cmath> #include "tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_buffer.h" #include "tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_summary.h" diff --git a/tensorflow/contrib/cmake/CMakeLists.txt b/tensorflow/contrib/cmake/CMakeLists.txt index af7b4fb386..83b405fd4e 100644 --- a/tensorflow/contrib/cmake/CMakeLists.txt +++ b/tensorflow/contrib/cmake/CMakeLists.txt @@ -229,6 +229,12 @@ if (tensorflow_ENABLE_GPU) endif() endif() +# Find python executable +include(FindPythonInterp) +if(NOT ${PYTHONINTERP_FOUND}) + message(FATAL_ERROR "CMake was unable to find a python interpreter.") +endif() + # Let's get to work! include(tf_core_framework.cmake) # NOTE: Disabled until issue #3996 is fixed. diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake index 9e2eb71b4c..bfd6a18c78 100755 --- a/tensorflow/contrib/cmake/tf_python.cmake +++ b/tensorflow/contrib/cmake/tf_python.cmake @@ -27,7 +27,6 @@ # 1. Resolve the installed version of Python (for Python.h and python). # TODO(mrry): Parameterize the build script to enable Python 3 building. -include(FindPythonInterp) if(NOT PYTHON_INCLUDE_DIR) set(PYTHON_NOT_FOUND false) exec_program("${PYTHON_EXECUTABLE}" diff --git a/tensorflow/contrib/cmake/tf_shared_lib.cmake b/tensorflow/contrib/cmake/tf_shared_lib.cmake index 47289fd9d2..9385ac52e9 100644 --- a/tensorflow/contrib/cmake/tf_shared_lib.cmake +++ b/tensorflow/contrib/cmake/tf_shared_lib.cmake @@ -82,6 +82,13 @@ target_link_libraries(tensorflow PRIVATE tf_protos_cc ) +# There is a bug in GCC 5 resulting in undefined reference to a __cpu_model function when +# linking to the tensorflow library. Adding the following libraries fixes it. +# See issue on github: https://github.com/tensorflow/tensorflow/issues/9593 +if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 5.0) + target_link_libraries(tensorflow PRIVATE gcc_s gcc) +endif() + if(WIN32) add_dependencies(tensorflow tensorflow_static) endif(WIN32) diff --git a/tensorflow/contrib/cmake/tools/create_def_file.py b/tensorflow/contrib/cmake/tools/create_def_file.py index 9bd287d0d7..b16a5eadb0 100644 --- a/tensorflow/contrib/cmake/tools/create_def_file.py +++ b/tensorflow/contrib/cmake/tools/create_def_file.py @@ -44,7 +44,7 @@ UNDNAME = "undname.exe" DUMPBIN = "dumpbin.exe" # Exclude if matched -EXCLUDE_RE = re.compile(r"deleting destructor|::internal::") +EXCLUDE_RE = re.compile(r"RTTI|deleting destructor|::internal::") # Include if matched before exclude INCLUDEPRE_RE = re.compile(r"google::protobuf::internal::ExplicitlyConstructed|" @@ -141,17 +141,17 @@ def main(): continue if not INCLUDE_RE.search(line): continue - + if "deleting destructor" in line: # Some of the symbols convered by INCLUDEPRE_RE export deleting # destructor symbols, which is a bad idea. # So we filter out such symbols here. continue - + if DATA_EXCLUDE_RE.search(line): def_fp.write("\t" + decorated + "\n") else: - def_fp.write("\t" + decorated + " DATA\n") + def_fp.write("\t" + decorated + " DATA\n") taken.add(decorated) exit_code = proc.wait() if exit_code != 0: diff --git a/tensorflow/contrib/distributions/python/ops/operator_pd_identity.py b/tensorflow/contrib/distributions/python/ops/operator_pd_identity.py index e6947bf609..4cee299790 100644 --- a/tensorflow/contrib/distributions/python/ops/operator_pd_identity.py +++ b/tensorflow/contrib/distributions/python/ops/operator_pd_identity.py @@ -115,7 +115,7 @@ class OperatorPDIdentity(operator_pd.OperatorPDBase): """Static check that the argument `x` is proper `shape`, `dtype`.""" # x is a typical argument e.g. to matmul or solve. In both cases, x should # have the same type/shape since this is a square matrix. These checks are - # ususally not needed since we ususally have some tensor backing this + # usually not needed since we usually have some tensor backing this # distribution, and the calls to tf.matmul do a shape/type check. # # Static checks only for efficiency, the identity should be fast. diff --git a/tensorflow/contrib/keras/python/keras/datasets/imdb.py b/tensorflow/contrib/keras/python/keras/datasets/imdb.py index bafd92aca6..5c087fe63f 100644 --- a/tensorflow/contrib/keras/python/keras/datasets/imdb.py +++ b/tensorflow/contrib/keras/python/keras/datasets/imdb.py @@ -41,7 +41,7 @@ def load_data(path='imdb.npz', num_words: max number of words to include. Words are ranked by how often they occur (in the training set) and only the most frequent words are kept - skip_top: skip the top N most frequently occuring words + skip_top: skip the top N most frequently occurring words (which may not be informative). maxlen: truncate sequences after this length. seed: random seed for sample shuffling. diff --git a/tensorflow/contrib/keras/python/keras/datasets/reuters.py b/tensorflow/contrib/keras/python/keras/datasets/reuters.py index 81e940a846..b1c22fee63 100644 --- a/tensorflow/contrib/keras/python/keras/datasets/reuters.py +++ b/tensorflow/contrib/keras/python/keras/datasets/reuters.py @@ -43,7 +43,7 @@ def load_data(path='reuters.npz', num_words: max number of words to include. Words are ranked by how often they occur (in the training set) and only the most frequent words are kept - skip_top: skip the top N most frequently occuring words + skip_top: skip the top N most frequently occurring words (which may not be informative). maxlen: truncate sequences after this length. test_split: Fraction of the dataset to be used as test data. diff --git a/tensorflow/contrib/keras/python/keras/engine/topology.py b/tensorflow/contrib/keras/python/keras/engine/topology.py index 0336fc4bf4..3d9ed51a1c 100644 --- a/tensorflow/contrib/keras/python/keras/engine/topology.py +++ b/tensorflow/contrib/keras/python/keras/engine/topology.py @@ -649,7 +649,7 @@ class Layer(tf_base_layers.Layer): 'but was passed an input_mask: ' + str(mask)) # masking not explicitly supported: return None as mask return None - # if masking is explictly supported, by default + # if masking is explicitly supported, by default # carry over the input mask return mask diff --git a/tensorflow/contrib/keras/python/keras/engine/training.py b/tensorflow/contrib/keras/python/keras/engine/training.py index ba6201713e..96d1c2f262 100644 --- a/tensorflow/contrib/keras/python/keras/engine/training.py +++ b/tensorflow/contrib/keras/python/keras/engine/training.py @@ -245,7 +245,7 @@ def _check_array_lengths(inputs, targets, weights): def _check_loss_and_target_compatibility(targets, loss_fns, output_shapes): - """Does validation on the compatiblity of targets and loss functions. + """Does validation on the compatibility of targets and loss functions. This helps prevent users from using loss functions incorrectly. diff --git a/tensorflow/contrib/keras/python/keras/layers/normalization.py b/tensorflow/contrib/keras/python/keras/layers/normalization.py index df77401aee..ea229fdce1 100644 --- a/tensorflow/contrib/keras/python/keras/layers/normalization.py +++ b/tensorflow/contrib/keras/python/keras/layers/normalization.py @@ -169,7 +169,7 @@ class BatchNormalization(Layer): def normalize_inference(): if needs_broadcasting: - # In this case we must explictly broadcast all parameters. + # In this case we must explicitly broadcast all parameters. broadcast_moving_mean = K.reshape(self.moving_mean, broadcast_shape) broadcast_moving_variance = K.reshape(self.moving_variance, broadcast_shape) diff --git a/tensorflow/contrib/keras/python/keras/models.py b/tensorflow/contrib/keras/python/keras/models.py index 52456a4bb5..1c041091fc 100644 --- a/tensorflow/contrib/keras/python/keras/models.py +++ b/tensorflow/contrib/keras/python/keras/models.py @@ -221,7 +221,7 @@ def load_model(filepath, custom_objects=None): obj: object, dict, or list. Returns: - The same structure, where occurences + The same structure, where occurrences of a custom object name have been replaced with the custom object. """ diff --git a/tensorflow/contrib/keras/python/keras/preprocessing/sequence.py b/tensorflow/contrib/keras/python/keras/preprocessing/sequence.py index 5a24a63b01..692a359ead 100644 --- a/tensorflow/contrib/keras/python/keras/preprocessing/sequence.py +++ b/tensorflow/contrib/keras/python/keras/preprocessing/sequence.py @@ -156,7 +156,7 @@ def skipgrams(sequence, of word indices (integers). If using a `sampling_table`, word indices are expected to match the rank of the words in a reference dataset (e.g. 10 would encode - the 10-th most frequently occuring token). + the 10-th most frequently occurring token). Note that index 0 is expected to be a non-word and will be skipped. vocabulary_size: int. maximum possible word index + 1 window_size: int. actually half-window. diff --git a/tensorflow/contrib/labeled_tensor/python/ops/core.py b/tensorflow/contrib/labeled_tensor/python/ops/core.py index 393c7f93f3..e6aded92ca 100644 --- a/tensorflow/contrib/labeled_tensor/python/ops/core.py +++ b/tensorflow/contrib/labeled_tensor/python/ops/core.py @@ -810,7 +810,7 @@ def axis_order_scope(axis_order=None): Example usage: with lt.axis_order_scope(['x', 'y', 'z']): - # result is guranteed to have the correct axis order + # result is guaranteed to have the correct axis order result = w + b You can nest scopes, in which case only the inner-most scope applies, e.g., diff --git a/tensorflow/contrib/layers/python/layers/feature_column.py b/tensorflow/contrib/layers/python/layers/feature_column.py index 4329e22f48..e1a27335ab 100644 --- a/tensorflow/contrib/layers/python/layers/feature_column.py +++ b/tensorflow/contrib/layers/python/layers/feature_column.py @@ -451,7 +451,7 @@ class _SparseColumn( return input_tensor def is_compatible(self, other_column): - """Check compatability of two sparse columns.""" + """Check compatibility of two sparse columns.""" if self.lookup_config and other_column.lookup_config: return self.lookup_config == other_column.lookup_config compatible = (self.length == other_column.length and @@ -2129,7 +2129,7 @@ class _CrossedColumn( "columns", "hash_bucket_size", "hash_key", "combiner", "ckpt_to_load_from", "tensor_name_in_ckpt" ])): - """Represents a cross transformation also known as conjuction or combination. + """Represents a cross transformation also known as conjunction or combination. Instances of this class are immutable. It crosses given `columns`. Crossed column output will be hashed to hash_bucket_size. diff --git a/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py b/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py index a9698163dd..74d8f24071 100644 --- a/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py +++ b/tensorflow/contrib/layers/python/layers/feature_column_ops_test.py @@ -1505,8 +1505,8 @@ class SequenceInputFromFeatureColumnTest(test.TestCase): expected_input_shape = [4, 3, embedding_dimension] self.assertAllEqual(expected_input_shape, model_input.shape) - # `ids_tensor` consists of 7 instances of <empty>, 3 occurences of "b", - # 2 occurences of "c" and 1 instance of "a". + # `ids_tensor` consists of 7 instances of <empty>, 3 occurrences of "b", + # 2 occurrences of "c" and 1 instance of "a". expected_gradient_values = sorted([0., 3., 2., 1.] * embedding_dimension) actual_gradient_values = np.sort(gradients[0].values, axis=None) self.assertAllClose(expected_gradient_values, actual_gradient_values) diff --git a/tensorflow/contrib/layers/python/layers/feature_column_test.py b/tensorflow/contrib/layers/python/layers/feature_column_test.py index aa3912a408..0f606a787d 100644 --- a/tensorflow/contrib/layers/python/layers/feature_column_test.py +++ b/tensorflow/contrib/layers/python/layers/feature_column_test.py @@ -173,7 +173,7 @@ class FeatureColumnTest(test.TestCase): for i in range(len(b1_value)): self.assertAllClose(b1_value[i], b2_value[i]) - # Test the case when a shared_embedding_name is explictly specified. + # Test the case when a shared_embedding_name is explicitly specified. d = fc.shared_embedding_columns( [a1, a2], dimension=4, diff --git a/tensorflow/contrib/layers/python/layers/layers.py b/tensorflow/contrib/layers/python/layers/layers.py index 32ca0c38d9..7a429f75bb 100644 --- a/tensorflow/contrib/layers/python/layers/layers.py +++ b/tensorflow/contrib/layers/python/layers/layers.py @@ -278,7 +278,7 @@ def _fused_batch_norm( trainable=trainable_gamma) # Create moving_mean and moving_variance variables and add them to the - # appropiate collections. + # appropriate collections. moving_mean_collections = utils.get_variable_collections( variables_collections, 'moving_mean') moving_mean_initializer = param_initializers.get( @@ -632,7 +632,7 @@ def batch_norm(inputs, trainable=trainable) # Create moving_mean and moving_variance variables and add them to the - # appropiate collections. We disable variable partitioning while creating + # appropriate collections. We disable variable partitioning while creating # them, because assign_moving_average is not yet supported for partitioned # variables. partitioner = variable_scope.get_variable_scope().partitioner @@ -1087,7 +1087,7 @@ def convolution2d_transpose( """Adds a convolution2d_transpose with an optional batch normalization layer. The function creates a variable called `weights`, representing the - kernel, that is convolved with the input. If `batch_norm_params` is `None`, a + kernel, that is convolved with the input. If `normalizer_fn` is `None`, a second variable called 'biases' is added to the result of the operation. Args: @@ -1847,9 +1847,9 @@ def separable_convolution2d( This op first performs a depthwise convolution that acts separately on channels, creating a variable called `depthwise_weights`. If `num_outputs` is not None, it adds a pointwise convolution that mixes channels, creating a - variable called `pointwise_weights`. Then, if `batch_norm_params` is None, - it adds bias to the result, creating a variable called 'biases', otherwise - it adds a batch normalization layer. It finally applies an activation function + variable called `pointwise_weights`. Then, if `normalizer_fn` is None, + it adds bias to the result, creating a variable called 'biases', otherwise, + the `normalizer_fn` is applied. It finally applies an activation function to produce the end result. Args: diff --git a/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator.py b/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator.py index d86ef8d477..fc092fccd7 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator.py +++ b/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator.py @@ -115,7 +115,7 @@ def dict_to_state_tuple(input_dict, cell): def _concatenate_context_input(sequence_input, context_input): - """Replicates `context_input` accross all timesteps of `sequence_input`. + """Replicates `context_input` across all timesteps of `sequence_input`. Expands dimension 1 of `context_input` then tiles it `sequence_length` times. This value is appended to `sequence_input` on dimension 2 and the result is @@ -177,7 +177,7 @@ def build_sequence_input(features, describing sequence features. All items in the set should be instances of classes derived from `FeatureColumn`. context_feature_columns: An iterable containing all the feature columns - describing context features i.e. features that apply accross all time + describing context features i.e. features that apply across all time steps. All items in the set should be instances of classes derived from `FeatureColumn`. weight_collections: List of graph collections to which weights are added. @@ -419,7 +419,7 @@ def _get_dynamic_rnn_model_fn( describing sequence features. All items in the set should be instances of classes derived from `FeatureColumn`. context_feature_columns: An iterable containing all the feature columns - describing context features, i.e., features that apply accross all time + describing context features, i.e., features that apply across all time steps. All items in the set should be instances of classes derived from `FeatureColumn`. predict_probabilities: A boolean indicating whether to predict probabilities @@ -603,7 +603,7 @@ class DynamicRnnEstimator(estimator.Estimator): describing sequence features. All items in the iterable should be instances of classes derived from `FeatureColumn`. context_feature_columns: An iterable containing all the feature columns - describing context features, i.e., features that apply accross all time + describing context features, i.e., features that apply across all time steps. All items in the set should be instances of classes derived from `FeatureColumn`. num_classes: the number of classes for a classification problem. Only diff --git a/tensorflow/contrib/learn/python/learn/estimators/head.py b/tensorflow/contrib/learn/python/learn/estimators/head.py index 25f2922bf8..52b4213463 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/head.py +++ b/tensorflow/contrib/learn/python/learn/estimators/head.py @@ -163,7 +163,7 @@ class Head(object): ModeFnOps.loss to compute and apply gradients. logits: logits `Tensor` to be used by the head. logits_input: `Tensor` from which to build logits, often needed when you - don't want to compute the logits. Typicaly this is the activation of the + don't want to compute the logits. Typically this is the activation of the last hidden layer in a DNN. Some heads (like the ones responsible for candidate sampling) intrinsically avoid computing full logits and only accepts logits_input. diff --git a/tensorflow/contrib/learn/python/learn/estimators/head_test.py b/tensorflow/contrib/learn/python/learn/estimators/head_test.py index d5777088de..f7934fc188 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/head_test.py +++ b/tensorflow/contrib/learn/python/learn/estimators/head_test.py @@ -895,7 +895,7 @@ class BinaryClassificationHeadTest(test.TestCase): _assert_summary_tags(self, ["loss"]) # logloss: z:label, x:logit # z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x)) - # expected_loss is (total_weighted_loss)/1 since htere is 1 nonzero + # expected_loss is (total_weighted_loss)/1 since there is 1 nonzero # weight. expected_loss = 0.062652342 _assert_metrics( diff --git a/tensorflow/contrib/learn/python/learn/estimators/kmeans.py b/tensorflow/contrib/learn/python/learn/estimators/kmeans.py index 5be07d271d..a473cf46d5 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/kmeans.py +++ b/tensorflow/contrib/learn/python/learn/estimators/kmeans.py @@ -29,6 +29,7 @@ from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import state_ops +from tensorflow.python.summary import summary from tensorflow.python.ops.control_flow_ops import with_dependencies from tensorflow.python.platform import tf_logging as logging from tensorflow.python.summary import summary diff --git a/tensorflow/contrib/learn/python/learn/estimators/state_saving_rnn_estimator.py b/tensorflow/contrib/learn/python/learn/estimators/state_saving_rnn_estimator.py index 02acd70812..9cb4c3515a 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/state_saving_rnn_estimator.py +++ b/tensorflow/contrib/learn/python/learn/estimators/state_saving_rnn_estimator.py @@ -144,7 +144,7 @@ def _prepare_features_for_sqss(features, labels, mode, describing sequence features. All items in the set should be instances of classes derived from `FeatureColumn`. context_feature_columns: An iterable containing all the feature columns - describing context features, i.e., features that apply accross all time + describing context features, i.e., features that apply across all time steps. All items in the set should be instances of classes derived from `FeatureColumn`. @@ -261,7 +261,7 @@ def _read_batch(cell, describing sequence features. All items in the set should be instances of classes derived from `FeatureColumn`. context_feature_columns: An iterable containing all the feature columns - describing context features, i.e., features that apply accross all time + describing context features, i.e., features that apply across all time steps. All items in the set should be instances of classes derived from `FeatureColumn`. num_threads: The Python integer number of threads enqueuing input examples @@ -420,7 +420,7 @@ def _get_rnn_model_fn(cell_type, describing sequence features. All items in the set should be instances of classes derived from `FeatureColumn`. context_feature_columns: An iterable containing all the feature columns - describing context features, i.e., features that apply accross all time + describing context features, i.e., features that apply across all time steps. All items in the set should be instances of classes derived from `FeatureColumn`. predict_probabilities: A boolean indicating whether to predict probabilities @@ -563,7 +563,7 @@ class StateSavingRnnEstimator(estimator.Estimator): describing sequence features. All items in the set should be instances of classes derived from `FeatureColumn`. context_feature_columns: An iterable containing all the feature columns - describing context features, i.e., features that apply accross all time + describing context features, i.e., features that apply across all time steps. All items in the set should be instances of classes derived from `FeatureColumn`. num_classes: The number of classes for categorization. Used only and diff --git a/tensorflow/contrib/learn/python/learn/monitors.py b/tensorflow/contrib/learn/python/learn/monitors.py index 50ad988b87..e97992fd20 100644 --- a/tensorflow/contrib/learn/python/learn/monitors.py +++ b/tensorflow/contrib/learn/python/learn/monitors.py @@ -474,7 +474,7 @@ class LoggingTrainable(EveryN): def every_n_step_begin(self, step): super(LoggingTrainable, self).every_n_step_begin(step) - # Get a list of trainable variables at the begining of every N steps. + # Get a list of trainable variables at the beginning of every N steps. # We cannot get this in __init__ because train_op has not been generated. trainables = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES, scope=self._scope) diff --git a/tensorflow/contrib/makefile/README.md b/tensorflow/contrib/makefile/README.md index f061b58775..9ba5c035a2 100644 --- a/tensorflow/contrib/makefile/README.md +++ b/tensorflow/contrib/makefile/README.md @@ -295,7 +295,7 @@ itself, you'll see it's broken up into host and target sections. If you are cross-compiling, you should look at customizing the target settings to match what you need for your desired system. -## Dependency Managment +## Dependency Management The Makefile loads in a list of dependencies stored in text files. These files are generated from the main Bazel build by running diff --git a/tensorflow/contrib/rnn/__init__.py b/tensorflow/contrib/rnn/__init__.py index d2f9cde4f6..2420c3e179 100644 --- a/tensorflow/contrib/rnn/__init__.py +++ b/tensorflow/contrib/rnn/__init__.py @@ -46,6 +46,7 @@ See @{$python/contrib.rnn} guide. @@IntersectionRNNCell @@PhasedLSTMCell @@HighwayWrapper +@@GLSTMCell ### RNNCell wrappers @@AttentionCellWrapper diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py index f4589e3d9e..89ad0fcd75 100644 --- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py +++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py @@ -194,6 +194,44 @@ class RNNCellTest(test.TestCase): m.name: 0.1 * np.ones([1, 4])}) self.assertEqual(len(res), 2) + def testBasicLSTMCellDimension0Error(self): + """Tests that dimension 0 in both(x and m) shape must be equal.""" + with self.test_session() as sess: + with variable_scope.variable_scope( + "root", initializer=init_ops.constant_initializer(0.5)): + num_units = 2 + state_size = num_units * 2 + batch_size = 3 + input_size = 4 + x = array_ops.zeros([batch_size, input_size]) + m = array_ops.zeros([batch_size - 1, state_size]) + with self.assertRaises(ValueError): + g, out_m = core_rnn_cell_impl.BasicLSTMCell( + num_units, state_is_tuple=False)(x, m) + sess.run([variables_lib.global_variables_initializer()]) + sess.run([g, out_m], + {x.name: 1 * np.ones([batch_size, input_size]), + m.name: 0.1 * np.ones([batch_size - 1, state_size])}) + + def testBasicLSTMCellStateSizeError(self): + """Tests that state_size must be num_units * 2.""" + with self.test_session() as sess: + with variable_scope.variable_scope( + "root", initializer=init_ops.constant_initializer(0.5)): + num_units = 2 + state_size = num_units * 3 # state_size must be num_units * 2 + batch_size = 3 + input_size = 4 + x = array_ops.zeros([batch_size, input_size]) + m = array_ops.zeros([batch_size, state_size]) + with self.assertRaises(ValueError): + g, out_m = core_rnn_cell_impl.BasicLSTMCell( + num_units, state_is_tuple=False)(x, m) + sess.run([variables_lib.global_variables_initializer()]) + sess.run([g, out_m], + {x.name: 1 * np.ones([batch_size, input_size]), + m.name: 0.1 * np.ones([batch_size, state_size])}) + def testBasicLSTMCellStateTupleType(self): with self.test_session(): with variable_scope.variable_scope( diff --git a/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py index 33fd35c1a3..334baa5f9c 100644 --- a/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py +++ b/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py @@ -904,6 +904,64 @@ class RNNCellTest(test.TestCase): # States are left untouched self.assertAllClose(res[2], res[3]) + def testGLSTMCell(self): + # Ensure that G-LSTM matches LSTM when number_of_groups = 1 + batch_size = 2 + num_units = 4 + number_of_groups = 1 + + with self.test_session() as sess: + with variable_scope.variable_scope( + "root1", initializer=init_ops.constant_initializer(0.5)): + x = array_ops.ones([batch_size, num_units]) + # When number_of_groups = 1, G-LSTM is equivalent to regular LSTM + gcell = rnn_cell.GLSTMCell(num_units=num_units, + number_of_groups=number_of_groups) + cell = core_rnn_cell_impl.LSTMCell(num_units=num_units) + self.assertTrue(isinstance(gcell.state_size, tuple)) + zero_state = gcell.zero_state(batch_size=batch_size, + dtype=dtypes.float32) + gh, gs = gcell(x, zero_state) + h, g = cell(x, zero_state) + + sess.run([variables.global_variables_initializer()]) + glstm_result = sess.run([gh, gs]) + lstm_result = sess.run([h, g]) + + self.assertAllClose(glstm_result[0], lstm_result[0], 1e-5) + self.assertAllClose(glstm_result[1], lstm_result[1], 1e-5) + + # Test that G-LSTM subgroup act like corresponding sub-LSTMs + batch_size = 2 + num_units = 4 + number_of_groups = 2 + + with self.test_session() as sess: + with variable_scope.variable_scope( + "root2", initializer=init_ops.constant_initializer(0.5)): + # input for G-LSTM with 2 groups + glstm_input = array_ops.ones([batch_size, num_units]) + gcell = rnn_cell.GLSTMCell(num_units=num_units, + number_of_groups=number_of_groups) + gcell_zero_state = gcell.zero_state(batch_size=batch_size, + dtype=dtypes.float32) + gh, gs = gcell(glstm_input, gcell_zero_state) + + # input for LSTM cell simulating single G-LSTM group + lstm_input = array_ops.ones([batch_size, num_units / number_of_groups]) + # note division by number_of_groups. This cell one simulates G-LSTM group + cell = core_rnn_cell_impl.LSTMCell(num_units= + int(num_units / number_of_groups)) + cell_zero_state = cell.zero_state(batch_size=batch_size, + dtype=dtypes.float32) + h, g = cell(lstm_input, cell_zero_state) + + sess.run([variables.global_variables_initializer()]) + [gh_res, h_res] = sess.run([gh, h]) + self.assertAllClose(gh_res[:, 0:int(num_units / number_of_groups)], + h_res, 1e-5) + self.assertAllClose(gh_res[:, int(num_units / number_of_groups):], + h_res, 1e-5) class LayerNormBasicLSTMCellTest(test.TestCase): diff --git a/tensorflow/contrib/rnn/python/ops/rnn_cell.py b/tensorflow/contrib/rnn/python/ops/rnn_cell.py index ad23e532b1..7a0f894404 100644 --- a/tensorflow/contrib/rnn/python/ops/rnn_cell.py +++ b/tensorflow/contrib/rnn/python/ops/rnn_cell.py @@ -1923,3 +1923,178 @@ class PhasedLSTMCell(core_rnn_cell.RNNCell): new_state = core_rnn_cell.LSTMStateTuple(new_c, new_h) return new_h, new_state + + +class GLSTMCell(core_rnn_cell.RNNCell): + """Group LSTM cell (G-LSTM). + + The implementation is based on: + + https://arxiv.org/abs/1703.10722 + + O. Kuchaiev and B. Ginsburg + "Factorization Tricks for LSTM Networks", ICLR 2017 workshop. + """ + + def __init__(self, num_units, initializer=None, num_proj=None, + number_of_groups=1, forget_bias=1.0, activation=math_ops.tanh, + reuse=None): + """Initialize the parameters of G-LSTM cell. + + Args: + num_units: int, The number of units in the G-LSTM cell + initializer: (optional) The initializer to use for the weight and + projection matrices. + num_proj: (optional) int, The output dimensionality for the projection + matrices. If None, no projection is performed. + number_of_groups: (optional) int, number of groups to use. + If `number_of_groups` is 1, then it should be equivalent to LSTM cell + forget_bias: Biases of the forget gate are initialized by default to 1 + in order to reduce the scale of forgetting at the beginning of + the training. + activation: Activation function of the inner states. + reuse: (optional) Python boolean describing whether to reuse variables + in an existing scope. If not `True`, and the existing scope already + has the given variables, an error is raised. + + Raises: + ValueError: If `num_units` or `num_proj` is not divisible by + `number_of_groups`. + """ + super(GLSTMCell, self).__init__(_reuse=reuse) + self._num_units = num_units + self._initializer = initializer + self._num_proj = num_proj + self._forget_bias = forget_bias + self._activation = activation + self._number_of_groups = number_of_groups + + if self._num_units % self._number_of_groups != 0: + raise ValueError("num_units must be divisible by number_of_groups") + if self._num_proj: + if self._num_proj % self._number_of_groups != 0: + raise ValueError("num_proj must be divisible by number_of_groups") + self._group_shape = [int(self._num_proj / self._number_of_groups), + int(self._num_units / self._number_of_groups)] + else: + self._group_shape = [int(self._num_units / self._number_of_groups), + int(self._num_units / self._number_of_groups)] + + if num_proj: + self._state_size = core_rnn_cell.LSTMStateTuple(num_units, num_proj) + self._output_size = num_proj + else: + self._state_size = core_rnn_cell.LSTMStateTuple(num_units, num_units) + self._output_size = num_units + + @property + def state_size(self): + return self._state_size + + @property + def output_size(self): + return self._output_size + + def _get_input_for_group(self, inputs, group_id, group_size): + """Slices inputs into groups to prepare for processing by cell's groups + + Args: + inputs: cell input or it's previous state, + a Tensor, 2D, [batch x num_units] + group_id: group id, a Scalar, for which to prepare input + group_size: size of the group + + Returns: + subset of inputs corresponding to group "group_id", + a Tensor, 2D, [batch x num_units/number_of_groups] + """ + return array_ops.slice(input_=inputs, + begin=[0, group_id * group_size], + size=[self._batch_size, group_size], + name=("GLSTM_group%d_input_generation" % group_id)) + + def call(self, inputs, state): + """Run one step of G-LSTM. + + Args: + inputs: input Tensor, 2D, [batch x num_units]. + state: this must be a tuple of state Tensors, both `2-D`, + with column sizes `c_state` and `m_state`. + + Returns: + A tuple containing: + + - A `2-D, [batch x output_dim]`, Tensor representing the output of the + G-LSTM after reading `inputs` when previous state was `state`. + Here output_dim is: + num_proj if num_proj was set, + num_units otherwise. + - LSTMStateTuple representing the new state of G-LSTM cell + after reading `inputs` when the previous state was `state`. + + Raises: + ValueError: If input size cannot be inferred from inputs via + static shape inference. + """ + (c_prev, m_prev) = state + + self._batch_size = inputs.shape[0].value or array_ops.shape(inputs)[0] + dtype = inputs.dtype + scope = vs.get_variable_scope() + with vs.variable_scope(scope, initializer=self._initializer): + i_parts = [] + j_parts = [] + f_parts = [] + o_parts = [] + + for group_id in range(self._number_of_groups): + with vs.variable_scope("group%d" % group_id): + x_g_id = array_ops.concat( + [self._get_input_for_group(inputs, group_id, + self._group_shape[0]), + self._get_input_for_group(m_prev, group_id, + self._group_shape[0])], axis=1) + R_k = _linear(x_g_id, 4 * self._group_shape[1], bias=False) + i_k, j_k, f_k, o_k = array_ops.split(R_k, 4, 1) + + i_parts.append(i_k) + j_parts.append(j_k) + f_parts.append(f_k) + o_parts.append(o_k) + + bi = vs.get_variable(name="bias_i", + shape=[self._num_units], + dtype=dtype, + initializer= + init_ops.constant_initializer(0.0, dtype=dtype)) + bj = vs.get_variable(name="bias_j", + shape=[self._num_units], + dtype=dtype, + initializer= + init_ops.constant_initializer(0.0, dtype=dtype)) + bf = vs.get_variable(name="bias_f", + shape=[self._num_units], + dtype=dtype, + initializer= + init_ops.constant_initializer(0.0, dtype=dtype)) + bo = vs.get_variable(name="bias_o", + shape=[self._num_units], + dtype=dtype, + initializer= + init_ops.constant_initializer(0.0, dtype=dtype)) + + i = nn_ops.bias_add(array_ops.concat(i_parts, axis=1), bi) + j = nn_ops.bias_add(array_ops.concat(j_parts, axis=1), bj) + f = nn_ops.bias_add(array_ops.concat(f_parts, axis=1), bf) + o = nn_ops.bias_add(array_ops.concat(o_parts, axis=1), bo) + + c = (math_ops.sigmoid(f + self._forget_bias) * c_prev + + math_ops.sigmoid(i) * math_ops.tanh(j)) + m = math_ops.sigmoid(o) * self._activation(c) + + if self._num_proj is not None: + with vs.variable_scope("projection"): + m = _linear(m, self._num_proj, bias=False) + + new_state = core_rnn_cell.LSTMStateTuple(c, m) + return m, new_state diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py index d3fc8d1d0d..8d1c0c59e0 100644 --- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py +++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py @@ -498,7 +498,7 @@ class AttentionWrapper(core_rnn_cell.RNNCell): if probability_fn is None: probability_fn = nn_ops.softmax else: - if not callable(cell_input_fn): + if not callable(probability_fn): raise TypeError( "probability_fn must be callable, saw type: %s" % type(probability_fn).__name__) diff --git a/tensorflow/contrib/session_bundle/example/export_half_plus_two.py b/tensorflow/contrib/session_bundle/example/export_half_plus_two.py index 08ca47058c..4a56509e59 100644 --- a/tensorflow/contrib/session_bundle/example/export_half_plus_two.py +++ b/tensorflow/contrib/session_bundle/example/export_half_plus_two.py @@ -97,7 +97,7 @@ def Export(export_dir, use_checkpoint_v2): } # Create two filename assets and corresponding tensors. - # TODO(b/26254158) Consider adding validation of file existance as well as + # TODO(b/26254158) Consider adding validation of file existence as well as # hashes (e.g. sha1) for consistency. original_filename1 = tf.constant("hello1.txt") tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, original_filename1) diff --git a/tensorflow/contrib/slim/python/slim/learning_test.py b/tensorflow/contrib/slim/python/slim/learning_test.py index cf3a878450..83d45f6f5a 100644 --- a/tensorflow/contrib/slim/python/slim/learning_test.py +++ b/tensorflow/contrib/slim/python/slim/learning_test.py @@ -840,7 +840,7 @@ class TrainTest(test.TestCase): # Initialize the variables. sess.run(variables_lib.global_variables_initializer()) - # Get the intial weights and biases values. + # Get the initial weights and biases values. weights_values, biases_values = sess.run([weights, biases]) self.assertGreater(np.linalg.norm(weights_values), 0) self.assertAlmostEqual(np.linalg.norm(biases_values), 0) diff --git a/tensorflow/contrib/sparsemax/python/kernel_tests/sparsemax_loss_test.py b/tensorflow/contrib/sparsemax/python/kernel_tests/sparsemax_loss_test.py index 89dbcd96f8..c8b4e472c9 100644 --- a/tensorflow/contrib/sparsemax/python/kernel_tests/sparsemax_loss_test.py +++ b/tensorflow/contrib/sparsemax/python/kernel_tests/sparsemax_loss_test.py @@ -159,7 +159,7 @@ class SparsemaxLossTest(test.TestCase): self.assertShapeEqual(q, tf_sparsemax_op) def _test_gradient_against_estimate(self, dtype, random, use_gpu): - """check sparsemax-loss Rop, aginst estimated-loss Rop""" + """check sparsemax-loss Rop, against estimated-loss Rop""" z = random.uniform(low=-3, high=3, size=(test_obs, 10)).astype(dtype) q = np.zeros((test_obs, 10)).astype(dtype) q[np.arange(0, test_obs), np.random.randint(0, 10, size=test_obs)] = 1 @@ -178,7 +178,7 @@ class SparsemaxLossTest(test.TestCase): self.assertLess(err, 1e-4) def _test_gradient_against_numpy(self, dtype, random, use_gpu): - """check sparsemax-loss Rop, aginst numpy Rop""" + """check sparsemax-loss Rop, against numpy Rop""" z = random.uniform(low=-3, high=3, size=(test_obs, 10)) q = np.zeros((test_obs, 10)) q[np.arange(0, test_obs), np.random.randint(0, 10, size=test_obs)] = 1 diff --git a/tensorflow/contrib/sparsemax/python/kernel_tests/sparsemax_test.py b/tensorflow/contrib/sparsemax/python/kernel_tests/sparsemax_test.py index eafac1b9ae..82d36ee9cb 100644 --- a/tensorflow/contrib/sparsemax/python/kernel_tests/sparsemax_test.py +++ b/tensorflow/contrib/sparsemax/python/kernel_tests/sparsemax_test.py @@ -188,7 +188,7 @@ class SparsemaxTest(test.TestCase): self.assertShapeEqual(z, tf_sparsemax_op) def _test_gradient_against_estimate(self, dtype, random, use_gpu): - """check sparsemax Rop, aginst estimated Rop""" + """check sparsemax Rop, against estimated Rop""" z = random.uniform(low=-3, high=3, size=(test_obs, 10)).astype(dtype) logits = array_ops.placeholder(dtype, name='z') @@ -204,7 +204,7 @@ class SparsemaxTest(test.TestCase): self.assertLess(err, 1e-4) def _test_gradient_against_numpy(self, dtype, random, use_gpu): - """check sparsemax Rop, aginst numpy Rop""" + """check sparsemax Rop, against numpy Rop""" z = random.uniform(low=-3, high=3, size=(test_obs, 10)).astype(dtype) logits = constant_op.constant(z, name='z') diff --git a/tensorflow/contrib/tensor_forest/client/random_forest.py b/tensorflow/contrib/tensor_forest/client/random_forest.py index 6ea1a10853..0ba636c697 100644 --- a/tensorflow/contrib/tensor_forest/client/random_forest.py +++ b/tensorflow/contrib/tensor_forest/client/random_forest.py @@ -79,7 +79,7 @@ class TensorForestLossHook(session_run_hook.SessionRunHook): current_loss = run_values.results['current_loss'] current_step = run_values.results['global_step'] self.steps += 1 - # Gaurd against the global step going backwards, which might happen + # Guard against the global step going backwards, which might happen # if we recover from something. if self.last_step == -1 or self.last_step > current_step: logging.info('TensorForestLossHook resetting last_step.') diff --git a/tensorflow/contrib/training/python/training/evaluation.py b/tensorflow/contrib/training/python/training/evaluation.py index 8a985fe0e2..bc0c60c85c 100644 --- a/tensorflow/contrib/training/python/training/evaluation.py +++ b/tensorflow/contrib/training/python/training/evaluation.py @@ -254,7 +254,7 @@ def checkpoints_iterator(checkpoint_dir, logging.info('Timed-out waiting for a checkpoint.') return if timeout_fn(): - # The timeout_fn indicated that we are truely done. + # The timeout_fn indicated that we are truly done. return else: # The timeout_fn indicated that more checkpoints may come. diff --git a/tensorflow/contrib/training/python/training/feeder.py b/tensorflow/contrib/training/python/training/feeder.py index a7f43cc07e..a5cd7c5c94 100644 --- a/tensorflow/contrib/training/python/training/feeder.py +++ b/tensorflow/contrib/training/python/training/feeder.py @@ -18,7 +18,7 @@ This helper handles the plumbing in order to set up a feeder task to push generated inputs to a pool of remote consumers; or to run an -identical feeding mechanism in a seperate thread in the same process. +identical feeding mechanism in a separate thread in the same process. Example usage for distributed feeding: @@ -331,7 +331,7 @@ class Feeder(object): they never close their queue. Second, they are added to the `Feeder.REMOTE_QUEUE_RUNNERS` collection, rather than `ops.GraphKeys.QUEUE_RUNNERS`, so they can be started/stopped - seperately. + separately. Args: queue: The queue. diff --git a/tensorflow/contrib/training/python/training/feeder_test.py b/tensorflow/contrib/training/python/training/feeder_test.py index 4d5cf9eff2..f3a2fee046 100644 --- a/tensorflow/contrib/training/python/training/feeder_test.py +++ b/tensorflow/contrib/training/python/training/feeder_test.py @@ -156,7 +156,7 @@ class FeederTest(test.TestCase): coord.join() def testFeederSeparateThread(self): - # Start a feeder on a seperate thread, but with a shared local queue + # Start a feeder on a separate thread, but with a shared local queue servers = self._create_local_cluster(worker=1) coord = coordinator.Coordinator() feed_thread = FeederThread(self, coord, servers, 'worker', 0) diff --git a/tensorflow/contrib/training/python/training/hparam.py b/tensorflow/contrib/training/python/training/hparam.py index 1d17782820..2e08593699 100644 --- a/tensorflow/contrib/training/python/training/hparam.py +++ b/tensorflow/contrib/training/python/training/hparam.py @@ -164,7 +164,7 @@ class HParams(object): import argparse parser = argparse.ArgumentParser(description='Train my model.') parser.add_argument('--hparams', type=str, - help='Comma seperated list of "name=value" pairs.') + help='Comma separated list of "name=value" pairs.') args = parser.parse_args() ... def my_program(): diff --git a/tensorflow/contrib/training/python/training/training_test.py b/tensorflow/contrib/training/python/training/training_test.py index e7c8fcd2a0..0af79cf2e3 100644 --- a/tensorflow/contrib/training/python/training/training_test.py +++ b/tensorflow/contrib/training/python/training/training_test.py @@ -508,7 +508,7 @@ class TrainTest(test.TestCase): # Initialize the variables. session.run(variables_lib2.global_variables_initializer()) - # Get the intial weights and biases values. + # Get the initial weights and biases values. weights_values, biases_values = session.run([weights, biases]) self.assertGreater(np.linalg.norm(weights_values), 0) self.assertAlmostEqual(np.linalg.norm(biases_values), 0) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 78b67941fe..484ef3bf4a 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -741,6 +741,7 @@ cc_library( "//tensorflow/core/kernels:mkl_concat_op", "//tensorflow/core/kernels:mkl_conv_op", "//tensorflow/core/kernels:mkl_fused_batch_norm_op", + "//tensorflow/core/kernels:mkl_identity_op", "//tensorflow/core/kernels:mkl_lrn_op", "//tensorflow/core/kernels:mkl_pooling_ops", "//tensorflow/core/kernels:mkl_relu_op", @@ -1287,7 +1288,10 @@ cc_library( ] + tf_additional_verbs_lib_defines(), linkopts = select({ "//tensorflow:freebsd": [], - "//conditions:default": ["-ldl"], + "//conditions:default": [ + "-ldl", + "-lpthread", + ], }), deps = tf_additional_lib_deps() + [ ":lib_hash_crc32c_accelerate_internal", @@ -2127,6 +2131,7 @@ tf_cc_test_mkl( "//tensorflow/core/kernels:mkl_concat_op", "//tensorflow/core/kernels:mkl_conv_op", "//tensorflow/core/kernels:mkl_fused_batch_norm_op", + "//tensorflow/core/kernels:mkl_identity_op", "//tensorflow/core/kernels:mkl_lrn_op", "//tensorflow/core/kernels:mkl_pooling_ops", "//tensorflow/core/kernels:mkl_relu_op", diff --git a/tensorflow/core/common_runtime/visitable_allocator.h b/tensorflow/core/common_runtime/visitable_allocator.h index c83e4a4e3a..8edf922d11 100644 --- a/tensorflow/core/common_runtime/visitable_allocator.h +++ b/tensorflow/core/common_runtime/visitable_allocator.h @@ -44,7 +44,7 @@ class VisitableAllocator : public Allocator { }; // Needed for cases when a VisitableAllocator gets wrapped for tracking. -// Multiple-inheritance is considered acceptible in this case because +// Multiple-inheritance is considered acceptable in this case because // VisitableAllocator is a pure virtual interface and only TrackingAllocator // has default implementation. class TrackingVisitableAllocator : public TrackingAllocator, diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_tensor_coding.cc b/tensorflow/core/distributed_runtime/rpc/grpc_tensor_coding.cc index ba206890ce..90e311a493 100644 --- a/tensorflow/core/distributed_runtime/rpc/grpc_tensor_coding.cc +++ b/tensorflow/core/distributed_runtime/rpc/grpc_tensor_coding.cc @@ -166,7 +166,7 @@ void EncodeTensorToByteBuffer(bool is_dead, const Tensor& val, (e_skeleton.size() + VarLengthEncodingSize(TensorProto::kTensorContentFieldNumber, tdata.size())); - string header; // All of RecvTensorRequest except the tensor() field + string header; // All of RecvTensorResponse except the tensor() field response.AppendToString(&header); size_t expected_size = diff --git a/tensorflow/core/framework/tensor.cc b/tensorflow/core/framework/tensor.cc index ecb9810d83..d049da1c9d 100644 --- a/tensorflow/core/framework/tensor.cc +++ b/tensorflow/core/framework/tensor.cc @@ -902,42 +902,27 @@ void Tensor::FillDescription(TensorDescription* description) const { } gtl::InlinedVector<int64, 4> Tensor::ComputeFlatInnerDims( - int64 num_out_dims) const { - if (num_out_dims == dims()) { - return shape_.dim_sizes(); - } + gtl::ArraySlice<int64> orig, int64 num_out_dims) { gtl::InlinedVector<int64, 4> out_dims(num_out_dims, 0); - const int64 num_elements = NumElements(); - int64 prod_out_dims = 1; - for (int64 out_dim = num_out_dims - 1; out_dim > 0; --out_dim) { - const int64 in_dim = out_dim + (dims() - num_out_dims); - out_dims[out_dim] = (in_dim >= dims() || in_dim < 0) ? 1 : dim_size(in_dim); - prod_out_dims *= out_dims[out_dim]; - } - if (prod_out_dims != 0) { - out_dims[0] = num_elements / prod_out_dims; - } else { - out_dims[0] = 0; + int64 offset = orig.size() - num_out_dims; + for (int64 out_dim = num_out_dims - 1; out_dim >= 0; --out_dim) { + const int64 in_dim = out_dim + offset; + out_dims[out_dim] = in_dim < 0 ? 1 : orig[in_dim]; + } + for (int64 in_dim = 0; in_dim < offset; ++in_dim) { + out_dims[0] *= orig[in_dim]; } return out_dims; } gtl::InlinedVector<int64, 4> Tensor::ComputeFlatOuterDims( - int64 num_out_dims) const { - if (num_out_dims == dims()) { - return shape_.dim_sizes(); - } + gtl::ArraySlice<int64> orig, int64 num_out_dims) { gtl::InlinedVector<int64, 4> out_dims(num_out_dims, 0); - const int64 num_elements = NumElements(); - int64 prod_out_dims = 1; - for (int64 out_dim = 0; out_dim < num_out_dims - 1; ++out_dim) { - out_dims[out_dim] = out_dim >= dims() ? 1 : dim_size(out_dim); - prod_out_dims *= out_dims[out_dim]; - } - if (prod_out_dims != 0) { - out_dims[num_out_dims - 1] = num_elements / prod_out_dims; - } else { - out_dims[num_out_dims - 1] = 0; + for (int64 out_dim = 0; out_dim <= num_out_dims - 1; ++out_dim) { + out_dims[out_dim] = out_dim >= orig.size() ? 1 : orig[out_dim]; + } + for (int64 in_dim = num_out_dims; in_dim < orig.size(); ++in_dim) { + out_dims[num_out_dims - 1] *= orig[in_dim]; } return out_dims; } diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h index 103da4c1b3..753548de84 100644 --- a/tensorflow/core/framework/tensor.h +++ b/tensorflow/core/framework/tensor.h @@ -304,6 +304,15 @@ class Tensor { template <typename T, size_t NDIMS = 2> typename TTypes<T, NDIMS>::Tensor flat_outer_dims(); + /// Returns the data as an Eigen::Tensor with NDIMS dimensions, collapsing the + /// first 'begin' Tensor dimensions into the first dimension of the result and + /// the Tensor dimensions of the last dims() - 'begin' - NDIMS into the last + /// dimension of the result. If 'begin' < 0 then the the |'begin'| leading + /// dimensions of size 1 will be added. If 'begin' + NDIMS > dims() then + /// 'begin' + NDIMS - dims() trailing dimensions of size 1 will be added. + template <typename T, size_t NDIMS = 3> + typename TTypes<T, NDIMS>::Tensor flat_inner_outer_dims(int64 begin); + template <typename T, size_t NDIMS> typename TTypes<T, NDIMS>::Tensor shaped(gtl::ArraySlice<int64> new_sizes); @@ -386,6 +395,9 @@ class Tensor { template <typename T, size_t NDIMS = 2> typename TTypes<T, NDIMS>::ConstTensor flat_outer_dims() const; + template <typename T, size_t NDIMS = 3> + typename TTypes<T, NDIMS>::Tensor flat_inner_outer_dims(int64 begin) const; + /// Render the first `max_entries` values in `*this` into a string. string SummarizeValue(int64 max_entries) const; @@ -429,10 +441,11 @@ class Tensor { gtl::ArraySlice<int64> new_sizes, Eigen::array<Eigen::DenseIndex, NDIMS>* dims) const; - // TODO(rmlarsen): These shouldn't hardcode '4' so that it lines up with // TensorShape's InlineVector. - gtl::InlinedVector<int64, 4> ComputeFlatInnerDims(int64 num_out_dims) const; - gtl::InlinedVector<int64, 4> ComputeFlatOuterDims(int64 num_out_dims) const; + static gtl::InlinedVector<int64, 4> ComputeFlatInnerDims( + gtl::ArraySlice<int64> orig, int64 num_out_dims); + static gtl::InlinedVector<int64, 4> ComputeFlatOuterDims( + gtl::ArraySlice<int64> orig, int64 num_out_dims); TensorShape shape_; TensorBuffer* buf_; @@ -638,22 +651,36 @@ typename TTypes<T>::ConstScalar Tensor::scalar() const { template <typename T, size_t NDIMS> typename TTypes<T, NDIMS>::Tensor Tensor::flat_inner_dims() { - return shaped<T, NDIMS>(ComputeFlatInnerDims(NDIMS)); + return shaped<T, NDIMS>(ComputeFlatInnerDims(shape_.dim_sizes(), NDIMS)); } template <typename T, size_t NDIMS> typename TTypes<T, NDIMS>::Tensor Tensor::flat_outer_dims() { - return shaped<T, NDIMS>(ComputeFlatOuterDims(NDIMS)); + return shaped<T, NDIMS>(ComputeFlatOuterDims(shape_.dim_sizes(), NDIMS)); +} + +template <typename T, size_t NDIMS> +typename TTypes<T, NDIMS>::Tensor Tensor::flat_inner_outer_dims(int64 begin) { + gtl::InlinedVector<int64,4> flat_outer = ComputeFlatOuterDims( + shape_.dim_sizes(), begin + NDIMS); + return shaped<T, NDIMS>(ComputeFlatInnerDims(flat_outer, NDIMS)); } template <typename T, size_t NDIMS> typename TTypes<T, NDIMS>::ConstTensor Tensor::flat_inner_dims() const { - return shaped<T, NDIMS>(ComputeFlatInnerDims(NDIMS)); + return shaped<T, NDIMS>(ComputeFlatInnerDims(shape_.dim_sizes(), NDIMS)); } template <typename T, size_t NDIMS> typename TTypes<T, NDIMS>::ConstTensor Tensor::flat_outer_dims() const { - return shaped<T, NDIMS>(ComputeFlatOuterDims(NDIMS)); + return shaped<T, NDIMS>(ComputeFlatOuterDims(shape_.dim_sizes(), NDIMS)); +} + +template <typename T, size_t NDIMS> +typename TTypes<T, NDIMS>::Tensor Tensor::flat_inner_outer_dims(int64 begin) const { + gtl::InlinedVector<int64,4> flat_outer = ComputeFlatOuterDims( + shape_.dim_sizes(), begin + NDIMS); + return shaped<T, NDIMS>(ComputeFlatInnerDims(flat_outer, NDIMS)); } inline Tensor::Tensor(const Tensor& other) diff --git a/tensorflow/core/framework/tensor_test.cc b/tensorflow/core/framework/tensor_test.cc index 424b4579d3..2d756ef2c3 100644 --- a/tensorflow/core/framework/tensor_test.cc +++ b/tensorflow/core/framework/tensor_test.cc @@ -202,11 +202,18 @@ TEST(Tensor_QInt32, Simple) { TestCopies<qint32>(t); } -TEST(Tensor_Float, Reshape) { - Tensor t(DT_FLOAT, TensorShape({2, 3, 4, 5})); - EXPECT_TRUE(t.shape().IsSameSize(TensorShape({2, 3, 4, 5}))); +class TensorReshapeTest : public ::testing::Test { +protected: + Tensor t; + Tensor zero_t; + + TensorReshapeTest() : t(DT_FLOAT, TensorShape({2, 3, 4, 5})), + zero_t(DT_FLOAT, TensorShape({3, 0, 2, 0, 5})) {} + + virtual void SetUp() { + EXPECT_TRUE(t.shape().IsSameSize(TensorShape({2, 3, 4, 5}))); + EXPECT_TRUE(zero_t.shape().IsSameSize(TensorShape({3, 0, 2, 0, 5}))); - { auto tensor = t.tensor<float, 4>(); EXPECT_EQ(2, tensor.dimension(0)); EXPECT_EQ(3, tensor.dimension(1)); @@ -217,6 +224,11 @@ TEST(Tensor_Float, Reshape) { tensor(0, 0, 0, 0) = 0.01f; tensor(1, 2, 3, 4) = 0.02f; } + +}; + +TEST_F(TensorReshapeTest, Reshape) { + LOG(INFO) << "shaped"; { auto shaped = t.shaped<float, 1>({120}); EXPECT_EQ(120, shaped.dimension(0)); @@ -248,6 +260,10 @@ TEST(Tensor_Float, Reshape) { EXPECT_EQ(shaped(0, 0, 0, 0), 0.01f); EXPECT_EQ(shaped(1, 2, 3, 4), 0.02f); } +} + +TEST_F(TensorReshapeTest, Flat) { + LOG(INFO) << "flat"; { auto flat = t.flat<float>(); EXPECT_EQ(flat(0), 0.01f); @@ -255,6 +271,10 @@ TEST(Tensor_Float, Reshape) { EXPECT_EQ(flat(0), 0.01f); EXPECT_EQ(flat(119), 0.02f); } +} + +TEST_F(TensorReshapeTest, FlatInnerDims) { + LOG(INFO) << "flat_inner_dims"; { auto flat_inner_dims = t.flat_inner_dims<float>(); EXPECT_EQ(24, flat_inner_dims.dimension(0)); @@ -263,13 +283,6 @@ TEST(Tensor_Float, Reshape) { EXPECT_EQ(flat_inner_dims(23, 4), 0.02f); } { - auto flat_outer_dims = t.flat_outer_dims<float>(); - EXPECT_EQ(2, flat_outer_dims.dimension(0)); - EXPECT_EQ(60, flat_outer_dims.dimension(1)); - EXPECT_EQ(flat_outer_dims(0, 0), 0.01f); - EXPECT_EQ(flat_outer_dims(1, 59), 0.02f); - } - { auto flat_inner_dims = t.flat_inner_dims<float, 3>(); EXPECT_EQ(6, flat_inner_dims.dimension(0)); EXPECT_EQ(4, flat_inner_dims.dimension(1)); @@ -278,14 +291,6 @@ TEST(Tensor_Float, Reshape) { EXPECT_EQ(flat_inner_dims(5, 3, 4), 0.02f); } { - auto flat_outer_dims = t.flat_outer_dims<float, 3>(); - EXPECT_EQ(2, flat_outer_dims.dimension(0)); - EXPECT_EQ(3, flat_outer_dims.dimension(1)); - EXPECT_EQ(20, flat_outer_dims.dimension(2)); - EXPECT_EQ(flat_outer_dims(0, 0, 0), 0.01f); - EXPECT_EQ(flat_outer_dims(1, 2, 19), 0.02f); - } - { auto flat_inner_dims = t.flat_inner_dims<float, 5>(); EXPECT_EQ(1, flat_inner_dims.dimension(0)); EXPECT_EQ(2, flat_inner_dims.dimension(1)); @@ -296,6 +301,44 @@ TEST(Tensor_Float, Reshape) { EXPECT_EQ(flat_inner_dims(0, 1, 2, 3, 4), 0.02f); } { + auto flat_inner_dims = zero_t.flat_inner_dims<float>(); + EXPECT_EQ(0, flat_inner_dims.dimension(0)); + EXPECT_EQ(5, flat_inner_dims.dimension(1)); + } + { + auto flat_inner_dims = zero_t.flat_inner_dims<float, 3>(); + EXPECT_EQ(0, flat_inner_dims.dimension(0)); + EXPECT_EQ(0, flat_inner_dims.dimension(1)); + EXPECT_EQ(5, flat_inner_dims.dimension(2)); + } + { + auto flat_inner_dims = zero_t.flat_inner_dims<float, 5>(); + EXPECT_EQ(3, flat_inner_dims.dimension(0)); + EXPECT_EQ(0, flat_inner_dims.dimension(1)); + EXPECT_EQ(2, flat_inner_dims.dimension(2)); + EXPECT_EQ(0, flat_inner_dims.dimension(3)); + EXPECT_EQ(5, flat_inner_dims.dimension(4)); + } +} + +TEST_F(TensorReshapeTest, FlatOuterDims) { + LOG(INFO) << "flat_outer_dims"; + { + auto flat_outer_dims = t.flat_outer_dims<float>(); + EXPECT_EQ(2, flat_outer_dims.dimension(0)); + EXPECT_EQ(60, flat_outer_dims.dimension(1)); + EXPECT_EQ(flat_outer_dims(0, 0), 0.01f); + EXPECT_EQ(flat_outer_dims(1, 59), 0.02f); + } + { + auto flat_outer_dims = t.flat_outer_dims<float, 3>(); + EXPECT_EQ(2, flat_outer_dims.dimension(0)); + EXPECT_EQ(3, flat_outer_dims.dimension(1)); + EXPECT_EQ(20, flat_outer_dims.dimension(2)); + EXPECT_EQ(flat_outer_dims(0, 0, 0), 0.01f); + EXPECT_EQ(flat_outer_dims(1, 2, 19), 0.02f); + } + { auto flat_outer_dims = t.flat_outer_dims<float, 5>(); EXPECT_EQ(2, flat_outer_dims.dimension(0)); EXPECT_EQ(3, flat_outer_dims.dimension(1)); @@ -305,8 +348,6 @@ TEST(Tensor_Float, Reshape) { EXPECT_EQ(flat_outer_dims(0, 0, 0, 0, 0), 0.01f); EXPECT_EQ(flat_outer_dims(1, 2, 3, 4, 0), 0.02f); } - - Tensor zero_t(DT_FLOAT, TensorShape({3, 0, 2, 0, 5})); { auto flat_outer_dims = zero_t.flat_outer_dims<float>(); EXPECT_EQ(3, flat_outer_dims.dimension(0)); @@ -326,24 +367,132 @@ TEST(Tensor_Float, Reshape) { EXPECT_EQ(0, flat_outer_dims.dimension(3)); EXPECT_EQ(5, flat_outer_dims.dimension(4)); } +} + +TEST_F(TensorReshapeTest, FlatInnerOuterDims) { + LOG(INFO) << "flat_inner_outer_dims"; { - auto flat_inner_dims = zero_t.flat_inner_dims<float>(); - EXPECT_EQ(0, flat_inner_dims.dimension(0)); - EXPECT_EQ(5, flat_inner_dims.dimension(1)); + auto flat_inner_outer_dims = t.flat_inner_outer_dims<float, 4>(0); + EXPECT_EQ(2, flat_inner_outer_dims.dimension(0)); + EXPECT_EQ(3, flat_inner_outer_dims.dimension(1)); + EXPECT_EQ(4, flat_inner_outer_dims.dimension(2)); + EXPECT_EQ(5, flat_inner_outer_dims.dimension(3)); + EXPECT_EQ(flat_inner_outer_dims(0, 0, 0, 0), 0.01f); + EXPECT_EQ(flat_inner_outer_dims(1, 2, 3, 4), 0.02f); } { - auto flat_inner_dims = zero_t.flat_inner_dims<float, 3>(); - EXPECT_EQ(0, flat_inner_dims.dimension(0)); - EXPECT_EQ(0, flat_inner_dims.dimension(1)); - EXPECT_EQ(5, flat_inner_dims.dimension(2)); + auto flat_inner_outer_dims = t.flat_inner_outer_dims<float, 6>(-2); + EXPECT_EQ(1, flat_inner_outer_dims.dimension(0)); + EXPECT_EQ(1, flat_inner_outer_dims.dimension(1)); + EXPECT_EQ(2, flat_inner_outer_dims.dimension(2)); + EXPECT_EQ(3, flat_inner_outer_dims.dimension(3)); + EXPECT_EQ(4, flat_inner_outer_dims.dimension(4)); + EXPECT_EQ(5, flat_inner_outer_dims.dimension(5)); + EXPECT_EQ(flat_inner_outer_dims(0, 0, 0, 0, 0, 0), 0.01f); + EXPECT_EQ(flat_inner_outer_dims(0, 0, 1, 2, 3, 4), 0.02f); } { - auto flat_inner_dims = zero_t.flat_inner_dims<float, 5>(); - EXPECT_EQ(3, flat_inner_dims.dimension(0)); - EXPECT_EQ(0, flat_inner_dims.dimension(1)); - EXPECT_EQ(2, flat_inner_dims.dimension(2)); - EXPECT_EQ(0, flat_inner_dims.dimension(3)); - EXPECT_EQ(5, flat_inner_dims.dimension(4)); + auto flat_inner_outer_dims = t.flat_inner_outer_dims<float, 6>(0); + EXPECT_EQ(2, flat_inner_outer_dims.dimension(0)); + EXPECT_EQ(3, flat_inner_outer_dims.dimension(1)); + EXPECT_EQ(4, flat_inner_outer_dims.dimension(2)); + EXPECT_EQ(5, flat_inner_outer_dims.dimension(3)); + EXPECT_EQ(1, flat_inner_outer_dims.dimension(4)); + EXPECT_EQ(1, flat_inner_outer_dims.dimension(5)); + EXPECT_EQ(flat_inner_outer_dims(0, 0, 0, 0, 0, 0), 0.01f); + EXPECT_EQ(flat_inner_outer_dims(1, 2, 3, 4, 0, 0), 0.02f); + } + { + auto flat_inner_outer_dims = t.flat_inner_outer_dims<float, 8>(-2); + EXPECT_EQ(1, flat_inner_outer_dims.dimension(0)); + EXPECT_EQ(1, flat_inner_outer_dims.dimension(1)); + EXPECT_EQ(2, flat_inner_outer_dims.dimension(2)); + EXPECT_EQ(3, flat_inner_outer_dims.dimension(3)); + EXPECT_EQ(4, flat_inner_outer_dims.dimension(4)); + EXPECT_EQ(5, flat_inner_outer_dims.dimension(5)); + EXPECT_EQ(1, flat_inner_outer_dims.dimension(6)); + EXPECT_EQ(1, flat_inner_outer_dims.dimension(7)); + EXPECT_EQ(flat_inner_outer_dims(0, 0, 0, 0, 0, 0, 0, 0), 0.01f); + EXPECT_EQ(flat_inner_outer_dims(0, 0, 1, 2, 3, 4, 0, 0), 0.02f); + } + { + auto flat_inner_outer_dims = t.flat_inner_outer_dims<float, 3>(1); + EXPECT_EQ(6, flat_inner_outer_dims.dimension(0)); + EXPECT_EQ(4, flat_inner_outer_dims.dimension(1)); + EXPECT_EQ(5, flat_inner_outer_dims.dimension(2)); + EXPECT_EQ(flat_inner_outer_dims(0, 0, 0), 0.01f); + EXPECT_EQ(flat_inner_outer_dims(5, 3, 4), 0.02f); + } + { + auto flat_inner_outer_dims = t.flat_inner_outer_dims<float, 5>(1); + EXPECT_EQ(6, flat_inner_outer_dims.dimension(0)); + EXPECT_EQ(4, flat_inner_outer_dims.dimension(1)); + EXPECT_EQ(5, flat_inner_outer_dims.dimension(2)); + EXPECT_EQ(1, flat_inner_outer_dims.dimension(3)); + EXPECT_EQ(1, flat_inner_outer_dims.dimension(4)); + EXPECT_EQ(flat_inner_outer_dims(0, 0, 0, 0, 0), 0.01f); + EXPECT_EQ(flat_inner_outer_dims(5, 3, 4, 0, 0), 0.02f); + } + { + auto flat_inner_outer_dims = t.flat_inner_outer_dims<float, 3>(0); + EXPECT_EQ(2, flat_inner_outer_dims.dimension(0)); + EXPECT_EQ(3, flat_inner_outer_dims.dimension(1)); + EXPECT_EQ(20, flat_inner_outer_dims.dimension(2)); + EXPECT_EQ(flat_inner_outer_dims(0, 0, 0), 0.01f); + EXPECT_EQ(flat_inner_outer_dims(1, 2, 19), 0.02f); + } + { + auto flat_inner_outer_dims = t.flat_inner_outer_dims<float, 5>(-2); + EXPECT_EQ(1, flat_inner_outer_dims.dimension(0)); + EXPECT_EQ(1, flat_inner_outer_dims.dimension(1)); + EXPECT_EQ(2, flat_inner_outer_dims.dimension(2)); + EXPECT_EQ(3, flat_inner_outer_dims.dimension(3)); + EXPECT_EQ(20, flat_inner_outer_dims.dimension(4)); + EXPECT_EQ(flat_inner_outer_dims(0, 0, 0, 0, 0), 0.01f); + EXPECT_EQ(flat_inner_outer_dims(0, 0, 1, 2, 19), 0.02f); + } + { + auto flat_inner_outer_dims = t.flat_inner_outer_dims<float, 2>(1); + EXPECT_EQ(6, flat_inner_outer_dims.dimension(0)); + EXPECT_EQ(20, flat_inner_outer_dims.dimension(1)); + EXPECT_EQ(flat_inner_outer_dims(0, 0), 0.01f); + EXPECT_EQ(flat_inner_outer_dims(5, 19), 0.02f); + } + { + auto flat_inner_outer_dims = zero_t.flat_inner_outer_dims<float, 2>(0); + EXPECT_EQ(3, flat_inner_outer_dims.dimension(0)); + EXPECT_EQ(0, flat_inner_outer_dims.dimension(1)); + } + { + auto flat_inner_outer_dims = zero_t.flat_inner_outer_dims<float, 3>(0); + EXPECT_EQ(3, flat_inner_outer_dims.dimension(0)); + EXPECT_EQ(0, flat_inner_outer_dims.dimension(1)); + EXPECT_EQ(0, flat_inner_outer_dims.dimension(2)); + } + { + auto flat_inner_outer_dims = zero_t.flat_inner_outer_dims<float, 5>(0); + EXPECT_EQ(3, flat_inner_outer_dims.dimension(0)); + EXPECT_EQ(0, flat_inner_outer_dims.dimension(1)); + EXPECT_EQ(2, flat_inner_outer_dims.dimension(2)); + EXPECT_EQ(0, flat_inner_outer_dims.dimension(3)); + EXPECT_EQ(5, flat_inner_outer_dims.dimension(4)); + } + { + auto flat_inner_outer_dims = zero_t.flat_inner_outer_dims<float, 2>(3); + EXPECT_EQ(0, flat_inner_outer_dims.dimension(0)); + EXPECT_EQ(5, flat_inner_outer_dims.dimension(1)); + } + { + auto flat_inner_outer_dims = zero_t.flat_inner_outer_dims<float, 3>(2); + EXPECT_EQ(0, flat_inner_outer_dims.dimension(0)); + EXPECT_EQ(0, flat_inner_outer_dims.dimension(1)); + EXPECT_EQ(5, flat_inner_outer_dims.dimension(2)); + } + { + auto flat_inner_outer_dims = zero_t.flat_inner_outer_dims<float, 3>(1); + EXPECT_EQ(0, flat_inner_outer_dims.dimension(0)); + EXPECT_EQ(2, flat_inner_outer_dims.dimension(1)); + EXPECT_EQ(0, flat_inner_outer_dims.dimension(2)); } } diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc index 09b632a165..94741a11ff 100644 --- a/tensorflow/core/graph/mkl_layout_pass.cc +++ b/tensorflow/core/graph/mkl_layout_pass.cc @@ -35,6 +35,7 @@ limitations under the License. #include "tensorflow/core/lib/gtl/map_util.h" #include "tensorflow/core/lib/hash/hash.h" #include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/util/tensor_format.h" #include "tensorflow/core/graph/mkl_layout_pass.h" #include "tensorflow/core/util/mkl_util.h" @@ -272,6 +273,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass { csinfo_.conv2d_grad_filter = "Conv2DBackpropFilter"; csinfo_.fused_batch_norm = "FusedBatchNorm"; csinfo_.fused_batch_norm_grad = "FusedBatchNormGrad"; + csinfo_.identity = "Identity"; csinfo_.lrn = "LRN"; csinfo_.lrn_grad = "LRNGrad"; csinfo_.matmul = "MatMul"; @@ -280,51 +282,75 @@ class MklLayoutRewritePass : public GraphOptimizationPass { csinfo_.mkl_conv2d = "_MklConv2D"; csinfo_.mkl_conv2d_with_bias = "_MklConv2DWithBias"; csinfo_.mkl_conv2d_with_bias_backprop_bias = - "_MklConv2DWithBiasBackpropBias"; - csinfo_.relu = "Relu"; - csinfo_.reshape = "Reshape"; - csinfo_.relu_grad = "ReluGrad"; - csinfo_.split = "Split"; + "_MklConv2DWithBiasBackpropBias"; + csinfo_.relu = "Relu"; + csinfo_.relu_grad = "ReluGrad"; + csinfo_.reshape = "Reshape"; + csinfo_.split = "Split"; // NOTE: names are alphabetically sorted. - rinfo_.push_back({csinfo_.avg_pool, GetMklOpName(csinfo_.avg_pool), 1, - CopyAttrsPooling, AlwaysRewrite}); + rinfo_.push_back({csinfo_.avg_pool, + GetMklOpName(csinfo_.avg_pool), + CopyAttrsPooling, AlwaysRewrite, nullptr}); rinfo_.push_back({csinfo_.avg_pool_grad, - GetMklOpName(csinfo_.avg_pool_grad), 2, CopyAttrsPooling, - AlwaysRewrite}); - rinfo_.push_back({csinfo_.concat, GetMklOpName(csinfo_.concat), 0, - CopyAttrsConcat, AlwaysRewrite}); - rinfo_.push_back({csinfo_.concatv2, GetMklOpName(csinfo_.concatv2), 0, - CopyAttrsConcatV2, AlwaysRewrite}); - rinfo_.push_back({csinfo_.conv2d, GetMklOpName(csinfo_.conv2d), 2, - CopyAttrsConv2D, AlwaysRewrite}); + GetMklOpName(csinfo_.avg_pool_grad), + CopyAttrsPooling, AlwaysRewrite, nullptr}); + // BiasAddGrad gets written into Conv2DWithBiasBackpropBias depending + // on if context contains Conv2D. + rinfo_.push_back({csinfo_.bias_add_grad, + csinfo_.mkl_conv2d_with_bias_backprop_bias, + CopyAttrsBiasAddGrad, ContextMatchRewrite, + &biasaddgrad_conv2dwithbias_context_}); + // BiasAddGrad gets written into BiasAddGrad depending on if context + // contains MatMul. + rinfo_.push_back({csinfo_.bias_add_grad, csinfo_.matmul, + CopyAttrsBiasAddGrad, ContextMatchRewrite, + &biasaddgrad_matmul_context_}); + rinfo_.push_back({csinfo_.concat, + GetMklOpName(csinfo_.concat), + CopyAttrsConcat, AlwaysRewrite, nullptr}); + rinfo_.push_back({csinfo_.concatv2, + GetMklOpName(csinfo_.concatv2), + CopyAttrsConcatV2, AlwaysRewrite, nullptr}); + rinfo_.push_back({csinfo_.conv2d, + GetMklOpName(csinfo_.conv2d), + CopyAttrsConv2D, AlwaysRewrite, nullptr}); rinfo_.push_back({csinfo_.conv2d_grad_filter, - GetMklOpName(csinfo_.conv2d_grad_filter), 3, - CopyAttrsConv2D, AlwaysRewrite}); + GetMklOpName(csinfo_.conv2d_grad_filter), + CopyAttrsConv2D, AlwaysRewrite, nullptr}); rinfo_.push_back({csinfo_.conv2d_grad_input, - GetMklOpName(csinfo_.conv2d_grad_input), 3, - CopyAttrsConv2D, AlwaysRewrite}); + GetMklOpName(csinfo_.conv2d_grad_input), + CopyAttrsConv2D, AlwaysRewrite, nullptr}); rinfo_.push_back({csinfo_.fused_batch_norm, - GetMklOpName(csinfo_.fused_batch_norm), 5, - CopyAttrsFusedBatchNorm, AlwaysRewrite}); + GetMklOpName(csinfo_.fused_batch_norm), + CopyAttrsFusedBatchNorm, AlwaysRewrite, nullptr}); rinfo_.push_back({csinfo_.fused_batch_norm_grad, - GetMklOpName(csinfo_.fused_batch_norm_grad), 5, - CopyAttrsFusedBatchNorm, AlwaysRewrite}); - rinfo_.push_back({csinfo_.lrn, GetMklOpName(csinfo_.lrn), 1, CopyAttrsLRN, - AlwaysRewrite}); - rinfo_.push_back({csinfo_.lrn_grad, GetMklOpName(csinfo_.lrn_grad), 3, - CopyAttrsLRN, AlwaysRewrite}); - rinfo_.push_back({csinfo_.max_pool, GetMklOpName(csinfo_.max_pool), 1, - CopyAttrsPooling, AlwaysRewrite}); + GetMklOpName(csinfo_.fused_batch_norm_grad), + CopyAttrsFusedBatchNorm, AlwaysRewrite, nullptr}); + rinfo_.push_back({csinfo_.identity, + GetMklOpName(csinfo_.identity), + CopyAttrsIdentity, AlwaysRewrite, nullptr}); + rinfo_.push_back({csinfo_.lrn, + GetMklOpName(csinfo_.lrn), + CopyAttrsLRN, AlwaysRewrite, nullptr}); + rinfo_.push_back({csinfo_.lrn_grad, + GetMklOpName(csinfo_.lrn_grad), + CopyAttrsLRN, AlwaysRewrite, nullptr}); + rinfo_.push_back({csinfo_.max_pool, + GetMklOpName(csinfo_.max_pool), + CopyAttrsPooling, NonDepthBatchWisePoolRewrite, nullptr}); rinfo_.push_back({csinfo_.max_pool_grad, - GetMklOpName(csinfo_.max_pool_grad), 3, CopyAttrsPooling, - AlwaysRewrite}); - rinfo_.push_back({csinfo_.relu, GetMklOpName(csinfo_.relu), 1, - CopyAttrsRelu, AlwaysRewrite}); - rinfo_.push_back({csinfo_.reshape, GetMklOpName(csinfo_.reshape), 2, - CopyAttrsReshape, AlwaysRewrite}); - - // TODO(inteltf): we do not support ReluGrad and BiasAddGrad yet. + GetMklOpName(csinfo_.max_pool_grad), + CopyAttrsPooling, AlwaysRewrite, nullptr}); + rinfo_.push_back({csinfo_.relu, + GetMklOpName(csinfo_.relu), + CopyAttrsRelu, AlwaysRewrite, nullptr}); + rinfo_.push_back({csinfo_.relu_grad, + GetMklOpName(csinfo_.relu_grad), + CopyAttrsRelu, AlwaysRewrite, nullptr}); + rinfo_.push_back({csinfo_.reshape, + GetMklOpName(csinfo_.reshape), + CopyAttrsReshape, AlwaysRewrite, nullptr}); // Add info about which ops to add workspace edge to and the slots. wsinfo_.push_back({csinfo_.lrn, csinfo_.lrn_grad, 0, 2, 1, 3}); @@ -338,8 +364,15 @@ class MklLayoutRewritePass : public GraphOptimizationPass { // maxhops in backward data-flow graph. Since input of forward nodes // (Conv2D) directly goes to backward nodes, we do not expect the // hop-distance would be more than few nodes. - cinfo_.push_back({csinfo_.bias_add_grad, csinfo_.mkl_conv2d_with_bias, - kNodeMergeContextMaxDepth}); + biasaddgrad_matmul_context_ = {csinfo_.bias_add_grad, csinfo_.matmul, + kNodeMergeContextMaxDepth}; + + biasaddgrad_conv2dwithbias_context_ = {csinfo_.bias_add_grad, + csinfo_.mkl_conv2d_with_bias, + kNodeMergeContextMaxDepth}; + + cinfo_.push_back(&biasaddgrad_matmul_context_); + cinfo_.push_back(&biasaddgrad_conv2dwithbias_context_); } // Standard interface to run pass @@ -354,7 +387,16 @@ class MklLayoutRewritePass : public GraphOptimizationPass { // @return true, if and only if graph is mutated; false otherwise. bool RunPass(std::unique_ptr<Graph>* g); - private: + /// Structure to specify the context information used in a node rewrite rule + typedef struct { + string node; // Name of the node to be rewritten + string fwd; // Name of the node in the forward pass that this node + // corresponds to + size_t max_hop; // Maximum number of hops the fwd is located + // from this node. If the fwd is farther than max_hop + // then we do not rewrite the node. + } ContextInfo; + /// Structure to specify the name of an original node, its new name after /// rewrite, the number of inputs to the original node, the function to /// be used to copy attributes for the op, and the rule (if any) which @@ -362,11 +404,12 @@ class MklLayoutRewritePass : public GraphOptimizationPass { typedef struct { string name; // Original name of op of the node in the graph string new_name; // New name of the op of the node in the graph - int num_ins; // The number of inputs to the original op type // A function handler to copy attributes from an old node to a new node. std::function<void(const Node*, NodeBuilder*)> copy_attrs; - std::function<bool(const Node*)> rewrite_rule; // A rule under which to - // rewrite this node. + // A rule under which to rewrite this node + std::function<bool(const Node*, const ContextInfo* c)> rewrite_rule; + // ContextInfo, if any, to be used for rewrite + ContextInfo* context; } RewriteInfo; /// Structure to specify a forward op, a backward op, and the slot numbers @@ -393,16 +436,6 @@ class MklLayoutRewritePass : public GraphOptimizationPass { string new_node; // Name of the node after merge } MergeInfo; - /// Structure to specify the context information used in a node rewrite rule - typedef struct { - string node; // Name of the node to be rewritten - string fwd; // Name of the node in the forward pass that this node - // corresponds to - size_t max_hop; // Maximum number of hops the fwd is located - // from this node. If the fwd is farther than max_hop - // then we do not rewrite the node. - } ContextInfo; - /// Structure to store all constant strings /// NOTE: names are alphabetically sorted. struct { @@ -417,6 +450,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass { string conv2d_grad_filter; string fused_batch_norm; string fused_batch_norm_grad; + string identity; string lrn; string lrn_grad; string matmul; @@ -427,10 +461,11 @@ class MklLayoutRewritePass : public GraphOptimizationPass { string mkl_conv2d_with_bias_backprop_bias; string relu; string relu_grad; - string split; string reshape; + string split; } csinfo_; + private: /// Maintain info about nodes to rewrite std::vector<RewriteInfo> rinfo_; @@ -441,7 +476,11 @@ class MklLayoutRewritePass : public GraphOptimizationPass { std::vector<MergeInfo> minfo_; /// Maintain info about nodes to rewrite - static std::vector<ContextInfo> cinfo_; + static std::vector<ContextInfo*> cinfo_; + + /// Context variables used in referencing rules + static ContextInfo biasaddgrad_matmul_context_; + static ContextInfo biasaddgrad_conv2dwithbias_context_; /// Hash table to maintain nodes visited in the graph. std::unordered_set<const Node*> visited_nodes_; @@ -464,19 +503,6 @@ class MklLayoutRewritePass : public GraphOptimizationPass { // Clear all visited nodes inline void UnMarkRewrittenNodes() { visited_nodes_.clear(); } - // Is this a graph node that can accept variable number of inputs? - // Return true if yes, false otherwise. - // - // Concat, Split are vararg nodes. - inline bool IsVarArgNode(Node* n) { - if (n->type_string() == csinfo_.concat || - n->type_string() == csinfo_.concatv2 || - n->type_string() == csinfo_.split) { - return true; - } - return false; - } - // Is OpDef::ArgDef a list type? It could be N * T or list(type). // Refer to opdef.proto for details of list type. inline bool ArgIsList(const OpDef::ArgDef& arg) const { @@ -510,6 +536,39 @@ class MklLayoutRewritePass : public GraphOptimizationPass { return string(kMklOpPrefix) + name; } + // Can op represented by node 'n' run on DEVICE_CPU? + // Op can run on CPU with MKL if the runtime assigned device or the + // user requested device contains device CPU, or both are empty. + bool CanOpRunOnCPUDevice(const Node* n) { + bool result = true; + string reason; + + // Substring that should be checked for in device name for CPU device. + const char* const kCPUDeviceSubStr = "cpu"; + + // If Op has been specifically assigned to a non-CPU device, then No. + if (!n->assigned_device_name().empty() && + !StringPiece(n->assigned_device_name()).contains(kCPUDeviceSubStr)) { + result = false; + reason = "Op has been assigned a runtime device that is not CPU."; + } + + // If user has specifically assigned this op to a non-CPU device, then No. + if (!n->def().device().empty() && + !StringPiece(n->def().device()).contains(kCPUDeviceSubStr)) { + result = false; + reason = "User has assigned a device that is not CPU."; + } + + if (result == false) { + VLOG(1) << "MklLayoutRewritePass: Skipping rewriting of the node " + << n->type_string() << ", reason: " << reason; + } + + // Otherwise Yes. + return result; + } + // Return a node that can be merged with input node 'n' // // @return pointer to the node if we can find such a @@ -538,13 +597,46 @@ class MklLayoutRewritePass : public GraphOptimizationPass { // Default rewrite rule to be used in scenario 1 for rewrite. // @return - true (since we want to always rewrite) - static bool AlwaysRewrite(const Node* n) { return true; } - // Rewrite rule that uses context-information for matching + static bool AlwaysRewrite(const Node* n, const ContextInfo* c = nullptr) { + return true; + } + + // Check if we are performing pooling on depth or batch. If it is, then we + // do not rewrite MaxPool node to Mkl version. + // @return - true (if it is not a depth/batch wise pooling case); + // false otherwise. + static bool NonDepthBatchWisePoolRewrite(const Node* n, + const ContextInfo* c) { + CHECK_NOTNULL(n); + + string data_format_str; + TensorFormat data_format; + std::vector<int32> ksize, strides; + CHECK_EQ(GetNodeAttr(n->def(), "ksize", &ksize).ok(), true); + CHECK_EQ(GetNodeAttr(n->def(), "strides", &strides).ok(), true); + CHECK_EQ(GetNodeAttr(n->def(), "data_format", &data_format_str).ok(), + true); + CHECK_EQ(FormatFromString(data_format_str, &data_format), true); + + // Condition that specifies non-batch-wise and non-depth-wise pooling. + if (GetTensorDim(ksize, data_format, 'N') == 1 && + GetTensorDim(strides, data_format, 'N') == 1 && + GetTensorDim(ksize, data_format, 'C') == 1 && + GetTensorDim(strides, data_format, 'C') == 1) { + return true; + } + + return false; + } + + // Rewrite rule that uses context-information for matching, // used in scenario 2. // // @input - Node 'n' for which to search for matching context - // @return - true if matching context is found; false otherwise. - static bool ContextMatchRewrite(const Node* n); + // @input - The context 'c' under which to rewrite + // @return - true if we can rewrite node under context 'c'; + // false otherwise. + static bool ContextMatchRewrite(const Node* n, const ContextInfo* c); // Helper function that searches the matching contextinfo for the node. // Implements depth-first search in the data dependence graph for the @@ -598,6 +690,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass { // node that we are constructing. // // @input g - input graph, + // @input orig_node - Original node that we are rewriting // @input inputs - inputs to old node that we are using for constructing // new inputs, // @input input_idx - the index in the 'inputs' vector pointing to the @@ -608,11 +701,10 @@ class MklLayoutRewritePass : public GraphOptimizationPass { // @output output_nodes - the list of new nodes creating Mkl tensors // // @return None - void GetNodesProducingMklTensorList( - std::unique_ptr<Graph>* g, - const gtl::InlinedVector<std::pair<Node*, int>, 4>& inputs, - int* input_idx, int list_length, - std::vector<NodeBuilder::NodeOut>* output_nodes); + void GetNodesProducingMklTensorList(std::unique_ptr<Graph>* g, + Node* orig_node, const gtl::InlinedVector<std::pair<Node*, int>, 4>& inputs, + int* input_idx, int list_length, + std::vector<NodeBuilder::NodeOut>* output_nodes); // Get a node that will feed an Mkl tensor to the new // node that we are constructing. The output node could be (1) 'n' @@ -620,6 +712,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass { // if 'n' is not an Mkl layer. // // @input g - input graph, + // @input orig_node - Original node that we are rewriting, // @input n - Node based on which we are creating Mkl node, // @input n_output_slot - the output slot of node 'n' // which is feeding to the node that we are constructing @@ -627,9 +720,8 @@ class MklLayoutRewritePass : public GraphOptimizationPass { // @output mkl_node_output_slot - the slot number of mkl_node that // will feed the tensor // @return None - void GetNodeProducingMklTensor(std::unique_ptr<Graph>* g, Node* n, - int n_output_slot, Node** mkl_node, - int* mkl_node_output_slot); + void GetNodeProducingMklTensor(std::unique_ptr<Graph>* g, Node* orig_node, + Node* n, int n_output_slot, Node** mkl_node, int* mkl_node_output_slot); // Setup new inputs using old inputs 'inputs' for the rewritten node in 'nb' // in graph 'g'. Original node is input in 'old_node'. Inputs to 'nb' are @@ -680,6 +772,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass { static void CopyAttrsConcatV2(const Node* orig_node, NodeBuilder* nb); static void CopyAttrsConv2D(const Node* orig_node, NodeBuilder* nb); static void CopyAttrsFusedBatchNorm(const Node* orig_node, NodeBuilder* nb); + static void CopyAttrsIdentity(const Node* orig_node, NodeBuilder* nb); static void CopyAttrsLRN(const Node* orig_node, NodeBuilder* nb); static void CopyAttrsPooling(const Node* orig_node, NodeBuilder* nb); static void CopyAttrsRelu(const Node* orig_node, NodeBuilder* nb); @@ -695,13 +788,18 @@ class MklLayoutRewritePass : public GraphOptimizationPass { Node* orig_node); }; -std::vector<MklLayoutRewritePass::ContextInfo> MklLayoutRewritePass::cinfo_; +MklLayoutRewritePass::ContextInfo + MklLayoutRewritePass::biasaddgrad_conv2dwithbias_context_; +MklLayoutRewritePass::ContextInfo + MklLayoutRewritePass::biasaddgrad_matmul_context_; +std::vector<MklLayoutRewritePass::ContextInfo*> MklLayoutRewritePass::cinfo_; -// We register Mkl rewrite pass for phase 1 in post rewrite group. +// We register Mkl rewrite pass for phase 1 in post partitioning group. // We register it here so that we get a complete picture of all users of Mkl // nodes. Do not change the ordering of the Mkl passes. -REGISTER_OPTIMIZATION(OptimizationPassRegistry::POST_REWRITE_FOR_EXEC, 1, - MklLayoutRewritePass); +const OptimizationPassRegistry::Grouping kMklLayoutRewritePassGroup = + OptimizationPassRegistry::POST_PARTITIONING; +REGISTER_OPTIMIZATION(kMklLayoutRewritePassGroup, 1, MklLayoutRewritePass); ////////////////////////////////////////////////////////////////////////// // Helper functions for creating new node @@ -737,27 +835,14 @@ void MklLayoutRewritePass::GetNodesProducingTFTensorList( while (list_length != 0) { CHECK_GT(list_length, 0); - CHECK_LE(*input_idx, inputs.size()); + CHECK_LT(*input_idx, inputs.size()); Node* n = inputs[*input_idx].first; int slot = inputs[*input_idx].second; - const OpDef::ArgDef& arg = n->op_def().output_arg(slot); - // If input node 'n' is producing a list/array output at output - // slot 'slot' then we need to find out the length of that list/array. - if (ArgIsList(arg)) { - int N = GetTensorListLength(arg, n); - CHECK_LE(N, list_length); - for (int j = 0; j < N; j++) { - output_nodes->push_back(NodeBuilder::NodeOut(n, slot)); - } - (*input_idx)++; - list_length -= N; - } else { - // But if input node 'n' is just producing a single tensor at - // output slot 'slot' then we just add that single node. - output_nodes->push_back(NodeBuilder::NodeOut(n, slot)); - (*input_idx)++; - list_length--; - } + // If input node 'n' is just producing a single tensor at + // output slot 'slot' then we just add that single node. + output_nodes->push_back(NodeBuilder::NodeOut(n, slot)); + (*input_idx)++; + list_length--; } } @@ -775,20 +860,39 @@ void MklLayoutRewritePass::GetDummyMklTensorNode(std::unique_ptr<Graph>* g, TensorShape dummy_shape({8}); dummy_shape.AsProto(proto.mutable_tensor_shape()); TF_CHECK_OK(NodeBuilder((*g)->NewName("DMT"), "Const") - .Attr("value", proto) - .Attr("dtype", dt) - .Device(orig_node->def().device()) // We place this node on - // the same device as the - // device of the original - // node. - .Finalize(&**g, out)); + .Attr("value", proto) + .Attr("dtype", dt) + .Device(orig_node->def().device()) // We place this node on + // the same device as the + // device of the original + // node. + .Finalize(&**g, out)); + + // If number of inputs to the original node is > 0, then we add + // control dependency between 1st input (index 0) of the original node and + // the dummy Mkl node. This is needed because control-flow ops such as Enter, + // Merge, etc, require frame_name of the dummy Mkl node to be same as the + // rewritten node. Adding control edge between 1st input of the original node + // and the dummy Mkl node ensures that the dummy node is in the same frame + // as the original node. Choosing 1st input is not necessary - any input of + // the original node is fine because all the inputs of a node are always in + // the same frame. + if (orig_node->num_inputs() > 0) { + Node* orig_input0 = nullptr; + TF_CHECK_OK(orig_node->input_node(0, + const_cast<const Node**>(&orig_input0))); + CHECK_NOTNULL((*g)->AddControlEdge(orig_input0, *out)); + } + (*out)->set_assigned_device_name(orig_node->assigned_device_name()); } void MklLayoutRewritePass::GetNodesProducingMklTensorList( std::unique_ptr<Graph>* g, - const gtl::InlinedVector<std::pair<Node*, int>, 4>& inputs, int* input_idx, - int list_length, std::vector<NodeBuilder::NodeOut>* output_nodes) { + Node* orig_node, + const gtl::InlinedVector<std::pair<Node*, int>, 4>& inputs, + int* input_idx, int list_length, + std::vector<NodeBuilder::NodeOut>* output_nodes) { CHECK_LT(*input_idx, inputs.size()); CHECK_GT(list_length, 0); CHECK_NOTNULL(output_nodes); @@ -796,38 +900,19 @@ void MklLayoutRewritePass::GetNodesProducingMklTensorList( while (list_length != 0) { CHECK_GT(list_length, 0); - CHECK_LE(*input_idx, inputs.size()); + CHECK_LT(*input_idx, inputs.size()); Node* n = inputs[*input_idx].first; int slot = inputs[*input_idx].second; - const OpDef::ArgDef& arg = n->op_def().output_arg(slot); - // We need to check first if the input edge is going to carry a - // single tensor or a list of tensors. If it is a list of tensors, - // then we need to create list of Mkl dummy nodes. - if (ArgIsList(arg)) { - // If input node 'n' is producing a list/array output at output - // slot 'slot' then we need to find out the length of that list/array. - int N = GetTensorListLength(arg, n); - CHECK_LE(N, list_length); - Node* mkl_node = nullptr; - int mkl_node_output_slot = 0; - // If it is a list, then create a list of Mkl dummy nodes. - for (int j = 0; j < N; j++) { - GetNodeProducingMklTensor(g, n, slot, &mkl_node, &mkl_node_output_slot); - output_nodes->push_back( - NodeBuilder::NodeOut(mkl_node, mkl_node_output_slot)); - } - (*input_idx)++; - list_length -= N; - } else { - // If it is not a list, then create a single Mkl tensor node. - Node* mkl_node = nullptr; - int mkl_node_output_slot = 0; - GetNodeProducingMklTensor(g, n, slot, &mkl_node, &mkl_node_output_slot); - output_nodes->push_back( - NodeBuilder::NodeOut(mkl_node, mkl_node_output_slot)); - (*input_idx)++; - list_length--; - } + // If 'n' is producing a single tensor, then create a single Mkl tensor + // node. + Node* mkl_node = nullptr; + int mkl_node_output_slot = 0; + GetNodeProducingMklTensor(g, orig_node, n, slot, &mkl_node, + &mkl_node_output_slot); + output_nodes->push_back(NodeBuilder::NodeOut(mkl_node, + mkl_node_output_slot)); + (*input_idx)++; + list_length--; } } @@ -835,9 +920,9 @@ void MklLayoutRewritePass::GetNodesProducingMklTensorList( // node that we are constructing. An input node could be (1) 'n' // if it is Mkl layer, or (2) a dummy node producing dummy Mkl tensor // if 'n' is not an Mkl layer. -void MklLayoutRewritePass::GetNodeProducingMklTensor( - std::unique_ptr<Graph>* g, Node* n, int n_output_slot, Node** mkl_node, - int* mkl_node_output_slot) { +void MklLayoutRewritePass::GetNodeProducingMklTensor(std::unique_ptr<Graph>* g, + Node* orig_node, Node* n, + int n_output_slot, Node** mkl_node, int* mkl_node_output_slot) { CHECK_NOTNULL(n); CHECK_NOTNULL(mkl_node); CHECK_NOTNULL(mkl_node_output_slot); @@ -860,7 +945,7 @@ void MklLayoutRewritePass::GetNodeProducingMklTensor( // to create a dummy node that will feed a dummy Mkl tensor to this node. // DummyMklTensor node has no input and generates only 1 output // (dummy Mkl tensor) as output slot number 0. - GetDummyMklTensorNode(g, mkl_node, n); + GetDummyMklTensorNode(g, mkl_node, orig_node); CHECK_NOTNULL(*mkl_node); *mkl_node_output_slot = 0; } @@ -926,16 +1011,16 @@ int MklLayoutRewritePass::SetUpContiguousInputs( if (ArgIsList(arg)) { std::vector<NodeBuilder::NodeOut> new_node_inputs; int N = GetTensorListLength(arg, old_node); - GetNodesProducingMklTensorList(g, old_node_inputs, &iidx, N, - &new_node_inputs); + GetNodesProducingMklTensorList(g, old_node, old_node_inputs, &iidx, + N, &new_node_inputs); nb->Input(new_node_inputs); nn_slot_idx++; } else { Node* mkl_node = nullptr; int mkl_node_output_slot = 0; - GetNodeProducingMklTensor(g, old_node_inputs[iidx].first, - old_node_inputs[iidx].second, &mkl_node, - &mkl_node_output_slot); + GetNodeProducingMklTensor(g, old_node, old_node_inputs[iidx].first, + old_node_inputs[iidx].second, + &mkl_node, &mkl_node_output_slot); nb->Input(mkl_node, mkl_node_output_slot); iidx++; nn_slot_idx++; @@ -1020,13 +1105,30 @@ void MklLayoutRewritePass::GetDummyWorkspaceTensorNode( TensorShape dummy_shape({1}); dummy_shape.AsProto(proto.mutable_tensor_shape()); TF_CHECK_OK(NodeBuilder((*g)->NewName("DMT"), "Const") - .Attr("value", proto) - .Attr("dtype", dt) - .Device(orig_node->def().device()) // We place this node on - // same the device as the - // device of the original - // node. - .Finalize(&**g, out)); + .Attr("value", proto) + .Attr("dtype", dt) + .Device(orig_node->def().device()) // We place this node on + // same the device as the + // device of the original + // node. + .Finalize(&**g, out)); + + // If number of inputs to the original node is > 0, then we add + // control dependency between 1st input (index 0) of the original node and + // the dummy Mkl node. This is needed because control-flow ops such as Enter, + // Merge, etc, require frame_name of the dummy Mkl node to be same as the + // rewritten node. Adding control edge between 1st input of the original node + // and the dummy Mkl node ensures that the dummy node is in the same frame + // as the original node. Choosing 1st input is not necessary - any input of + // the original node is fine because all the inputs of a node are always in + // the same frame. + if (orig_node->num_inputs() > 0) { + Node* orig_input0 = nullptr; + TF_CHECK_OK(orig_node->input_node(0, + const_cast<const Node**>(&orig_input0))); + CHECK_NOTNULL((*g)->AddControlEdge(orig_input0, *out)); + } + (*out)->set_assigned_device_name(orig_node->assigned_device_name()); } @@ -1179,6 +1281,16 @@ void MklLayoutRewritePass::CopyAttrsBiasAddGrad(const Node* orig_node, nb->Attr("data_format", data_format); } +void MklLayoutRewritePass::CopyAttrsIdentity(const Node* orig_node, + NodeBuilder* nb) { + DataType T; + + // Get all attributes from old node. + TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T)); + // Add attributes to new node. + nb->Attr("T", T); +} + void MklLayoutRewritePass::CopyAttrsLRN(const Node* orig_node, NodeBuilder* nb) { DataType T; @@ -1235,6 +1347,19 @@ void MklLayoutRewritePass::CopyAttrsRelu(const Node* orig_node, nb->Attr("T", T); } +void MklLayoutRewritePass::CopyAttrsReshape(const Node* orig_node, + NodeBuilder* nb) { + DataType T; + DataType Tshape; + + // Get all attributes from old node. + TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T)); + TF_CHECK_OK(GetNodeAttr(orig_node->def(), "Tshape", &Tshape)); + // Add attributes to new node. + nb->Attr("T", T); + nb->Attr("Tshape", Tshape); +} + void MklLayoutRewritePass::CopyAttrsSplit(const Node* orig_node, NodeBuilder* nb) { DataType T; @@ -1303,20 +1428,6 @@ void MklLayoutRewritePass::CopyAttrsFusedBatchNorm(const Node* orig_node, nb->Attr("is_training", is_training); } -void MklLayoutRewritePass::CopyAttrsReshape(const Node* orig_node, - NodeBuilder* nb) { - DataType T; - DataType Tshape; - - // Get all attributes from old node. - TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T)); - TF_CHECK_OK(GetNodeAttr(orig_node->def(), "Tshape", &Tshape)); - - // Add attributes to new node. - nb->Attr("T", T); - nb->Attr("Tshape", Tshape); -} - ////////////////////////////////////////////////////////////////////////// // Helper functions related to node merge pass ////////////////////////////////////////////////////////////////////////// @@ -1353,8 +1464,9 @@ Node* MklLayoutRewritePass::CheckForNodeMerge(const Node* a) const { continue; } + const int B_in = b->num_inputs(); gtl::InlinedVector<Node*, 4> b_control_edges; - gtl::InlinedVector<std::pair<Node*, int>, 4> b_in(N_in); + gtl::InlinedVector<std::pair<Node*, int>, 4> b_in(B_in); FillInputs(b, &b_control_edges, &b_in); // Shouldn't merge if a and b have different control edges. @@ -1438,7 +1550,7 @@ Status MklLayoutRewritePass::MergeNode(std::unique_ptr<Graph>* g, Node* succ, CHECK_EQ(succ->in_edges().size(), 2); Node* oper3_mkl = nullptr; // Mkl tensor corresponding to oper3 int oper3_mkl_slot = 0; // For dummy MKL tensor node, output slot is 0. - GetDummyMklTensorNode(g, &oper3_mkl, succ); // Get dummy Mkl tensor node + GetDummyMklTensorNode(g, &oper3_mkl, pred); // Get dummy Mkl tensor node // as BiasAdd does not have Mkl tensor as input. CHECK_NOTNULL(oper3_mkl); @@ -1483,9 +1595,38 @@ Status MklLayoutRewritePass::MergeNode(std::unique_ptr<Graph>* g, Node* succ, // Set the Mkl layer label for this op. new_node->AddAttr("_kernel", mkl_op_registry::kMklOpLabel); + // Incoming data edges from 'pred' node and 'succ' node to new 'new_node' + // node are already copied in BuildNode. We handle control edges now. + for (const Edge* e : pred->in_edges()) { + if (e->IsControlEdge()) { + CHECK_NOTNULL((*g)->AddControlEdge(e->src(), new_node)); + } + } + for (const Edge* e : succ->in_edges()) { + if (e->IsControlEdge()) { + CHECK_NOTNULL((*g)->AddControlEdge(e->src(), new_node)); + } + } + // Incoming edges are fixed, we will fix the outgoing edges now. + // First, we will fix outgoing control edges from 'pred' node. + // We don't need to handle outgoing data edges from 'pred' node + // because pred has only 1 output going to succ node (we enforced + // this check for merge already). + for (const Edge* e : pred->out_edges()) { + if (e->IsControlEdge()) { + CHECK_NOTNULL((*g)->AddControlEdge(new_node, e->dst())); + } + } + + // Second, we will fix outgoing control and data edges from 'succ' node. for (const Edge* e : succ->out_edges()) { - (*g)->AddEdge(new_node, e->src_output(), e->dst(), e->dst_input()); + if (e->IsControlEdge()) { + CHECK_NOTNULL((*g)->AddControlEdge(new_node, e->dst())); + } else { + CHECK_NOTNULL((*g)->AddEdge(new_node, e->src_output(), e->dst(), + e->dst_input())); + } } // Copy device assigned to old node to new node. @@ -1550,18 +1691,22 @@ Status MklLayoutRewritePass::RewriteNode(std::unique_ptr<Graph>* g, "data_format or T attribute or devices of BiasAddGrad and " "Conv2D do not match. Will skip node rewrite optimization"); } + } else if (orig_node->type_string() == csinfo_.bias_add_grad && + ri->new_name == csinfo_.matmul) { + // When BiasAddGrad has MatMul in context, we do not do any rewrite + // and leave BiasAddGrad as it is. But we check for this condition + // when we check for node rewrite rule. So we should not even come + // here for MatMul. So we will fail now. + return Status( + error::Code::INVALID_ARGUMENT, + "No rewrite is required for BiasAddGrad for MatMul context."); } } // Get all inputs. - const int num = orig_node->in_edges().size(); - // Check the number of inputs against the user-specified value for non-vararg - // nodes. - if (!IsVarArgNode(orig_node)) { - CHECK_EQ(num, ri->num_ins); - } + const int num_inputs = orig_node->in_edges().size(); gtl::InlinedVector<Node*, 4> control_edges; - gtl::InlinedVector<std::pair<Node*, int>, 4> inputs(num); + gtl::InlinedVector<std::pair<Node*, int>, 4> inputs(num_inputs); FillInputs(orig_node, &control_edges, &inputs); // Build new node. We use same name as original node, but change the op name. @@ -1596,8 +1741,15 @@ Status MklLayoutRewritePass::RewriteNode(std::unique_ptr<Graph>* g, TF_CHECK_OK(nb.Finalize(&**g, &new_node)); CHECK_NOTNULL(new_node); - // Incoming edges from 'orig_node' node to new 'new_node' node are already - // copied in BuildNode. Copy outgoing edges from 'orig_node' node to new + // Incoming data edges from 'orig_node' node to new 'new_node' node are + // already copied in BuildNode. We need to handle control edges now. + for (const Edge* e : orig_node->in_edges()) { + if (e->IsControlEdge()) { + CHECK_NOTNULL((*g)->AddControlEdge(e->src(), new_node)); + } + } + + // Copy outgoing edges from 'orig_node' node to new // 'new_node' node, since the output also follows same ordering among // Tensorflow tensors and Mkl tensors. We need to connect Tensorflow // tensors appropriately. Specifically, nth output of the original node @@ -1605,15 +1757,12 @@ Status MklLayoutRewritePass::RewriteNode(std::unique_ptr<Graph>* g, // of the tensors. For the contiguous ordering of the tensors, it will be n. // GetTensorDataIndex provides this mapping function. for (const Edge* e : orig_node->out_edges()) { - // We need to handle control-edges by using their original slot number. - // Generally, -1 is reserved for control slot. - if (e->src_output() < 0) { - (*g)->AddEdge(new_node, e->src_output(), e->dst(), e->dst_input()); + if (e->IsControlEdge()) { + CHECK_NOTNULL((*g)->AddControlEdge(new_node, e->dst())); } else { - (*g)->AddEdge( - new_node, - GetTensorDataIndex(e->src_output(), e->src()->num_outputs()), - e->dst(), e->dst_input()); + CHECK_NOTNULL((*g)->AddEdge(new_node, GetTensorDataIndex(e->src_output(), + e->src()->num_outputs()), + e->dst(), e->dst_input())); } } @@ -1640,8 +1789,8 @@ MklLayoutRewritePass::SearchMatchingContext(const Node* n, bool is_matching_cinfo_found = false; std::vector<const ContextInfo*> mci; for (auto ci = cinfo_.cbegin(); ci != cinfo_.cend(); ++ci) { - if (n->type_string() == ci->node) { - mci.push_back(&*ci); + if (n->type_string() == (*ci)->node) { + mci.push_back(*ci); is_matching_cinfo_found = true; } } @@ -1701,9 +1850,10 @@ MklLayoutRewritePass::SearchMatchingContext(const Node* n, return nullptr; } -bool MklLayoutRewritePass::ContextMatchRewrite(const Node* n) { +bool MklLayoutRewritePass::ContextMatchRewrite(const Node* n, + const ContextInfo* c) { const Node* fwd_node = nullptr; - return SearchMatchingContext(n, &fwd_node) != nullptr; + return SearchMatchingContext(n, &fwd_node) == c; } const MklLayoutRewritePass::RewriteInfo* @@ -1719,18 +1869,29 @@ MklLayoutRewritePass::CheckForNodeRewrite(const Node* n) const { return nullptr; } - if (!mkl_op_registry::IsMklOp(GetMklOpName(n->type_string()), T)) { - return nullptr; + // BiasAddGrad is not an Mkl layer, so we make an exception for it. + if (n->type_string() != csinfo_.bias_add_grad) { + if (!mkl_op_registry::IsMklOp(GetMklOpName(n->type_string()), T)) { + return nullptr; + } } // We support 2 types of node rewrites: - // 1. Rewriting BiasAddGrad depending on its context. + // 1. Rewriting BiasAddGrad depending on its MklConv2DWithBias context. // 2. Rewriting an op to Mkl op always // We return true if any of these 2 conditions is met. // Find matching RewriteInfo and then check that rewrite rule applies. for (auto ri = rinfo_.cbegin(); ri != rinfo_.cend(); ++ri) { - if (n->type_string().compare(ri->name) == 0 && ri->rewrite_rule(n)) { + if (n->type_string().compare(ri->name) == 0 && + ri->rewrite_rule(n, ri->context)) { + // If we are rewriting BiasAddGrad into BiasAddGrad for MatMul context, + // then we just return directly. + if (n->type_string() == csinfo_.bias_add_grad && + ri->context->fwd == csinfo_.matmul && + ri->new_name == csinfo_.bias_add_grad) { + return nullptr; + } return &*ri; } } @@ -1753,7 +1914,8 @@ bool MklLayoutRewritePass::RunPass(std::unique_ptr<Graph>* g) { GetReversePostOrder(**g, &order); // This will give us topological sort. for (Node* n : order) { - if (!n->IsOp()) { + // If node is not an op or it cannot run on CPU device, then skip. + if (!n->IsOp() || !CanOpRunOnCPUDevice(n)) { continue; } @@ -1801,18 +1963,31 @@ bool RunMklLayoutRewritePass(std::unique_ptr<Graph>* g) { return MklLayoutRewritePass().RunPass(g); } -Status MklLayoutRewritePass::Run(const GraphOptimizationPassOptions& options) { - if (options.graph == nullptr) { +Status MklLayoutRewritePass::Run( + const GraphOptimizationPassOptions& options) { + if (options.graph == nullptr && options.partition_graphs == nullptr) { return Status::OK(); } - // Get the ownership of graph - std::unique_ptr<Graph>* g = std::move(options.graph); - - RunPass(g); - - // Return the ownership of graph back - options.graph->reset(g->release()); + auto process_graph = [&](std::unique_ptr<Graph>* g) { + // Get the ownership of a graph + std::unique_ptr<Graph>* ng = std::move(g); + RunPass(ng); + // Return the ownership of a graph back + g->reset(ng->release()); + }; + + if (kMklLayoutRewritePassGroup != + OptimizationPassRegistry::POST_PARTITIONING) { + // For any pre-partitioning phase, a graph is stored in options.graph. + process_graph(options.graph); + } else { + // For post partitioning phase, graphs are stored in + // options.partition_graphs. + for (auto& pg : *options.partition_graphs) { + process_graph(&pg.second); + } + } return Status::OK(); } diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc index 6e72baf84e..3c4a5263af 100644 --- a/tensorflow/core/graph/mkl_layout_pass_test.cc +++ b/tensorflow/core/graph/mkl_layout_pass_test.cc @@ -39,7 +39,11 @@ limitations under the License. namespace tensorflow { namespace { -static void InitGraph(const string& s, Graph* graph) { +const char kCPUDevice[] = "/job:a/replica:0/task:0/cpu:0"; +const char kGPUDevice[] = "/job:a/replica:0/task:0/gpu:0"; + +static void InitGraph(const string& s, Graph* graph, + const string& device = kCPUDevice) { GraphDef graph_def; auto parser = protobuf::TextFormat::Parser(); @@ -47,14 +51,18 @@ static void InitGraph(const string& s, Graph* graph) { CHECK(parser.MergeFromString(s, &graph_def)) << s; GraphConstructorOptions opts; TF_CHECK_OK(ConvertGraphDefToGraph(opts, graph_def, graph)); + + for (Node* node : graph->nodes()) { + node->set_assigned_device_name(device); + } } class MklLayoutPassTest : public ::testing::Test { public: MklLayoutPassTest() : graph_(OpRegistry::Global()) {} - void InitGraph(const string& s) { - ::tensorflow::InitGraph(s, &graph_); + void InitGraph(const string& s, const string& device = kCPUDevice) { + ::tensorflow::InitGraph(s, &graph_, device); original_ = CanonicalGraphString(&graph_); } @@ -114,7 +122,8 @@ REGISTER_OP("InputList").Output("o: N * float").Attr("N: int").SetIsStateful(); REGISTER_OP("HalfInput").Output("o: half").SetIsStateful(); REGISTER_OP("Int32Input").Output("o: int32").SetIsStateful(); REGISTER_OP("_MklInput").Output("o: uint8").SetIsStateful(); -REGISTER_OP("_MklInput2").Output("o: uint8").Output("o1: uint8").SetIsStateful(); +REGISTER_OP("_MklInput2").Output("o: uint8") + .Output("o1: uint8").SetIsStateful(); ///////////////////////////////////////////////////////////////////// // Unit tests related to node merge optiimization @@ -162,8 +171,9 @@ TEST_F(MklLayoutPassTest, NodeMerge_Conv2DWithBias_Positive) { " input: ['E', 'Y']}"); EXPECT_EQ(DoMklLayoutOptimizationPass(), "A(Input);B(Input);D(Input);DMT/_0(Const);E(_MklConv2DWithBias);" - "M(_MklInput);N(_MklInput);Y(Input);Z(Sub)|A->E;B->E:1;D->E:2;" - "DMT/_0->E:5;E->Z;M->E:3;N->E:4;Y->Z:1"); + "M(_MklInput);N(_MklInput);Y(Input);Z(Sub)|A->E;" + "A:control->DMT/_0:control;B->E:1;D->E:2;DMT/_0->E:5;E->Z;M->E:3;" + "N->E:4;Y->Z:1"); } // C=_MklConv2D(A,M:1,B,N:1); E=BiasAdd(C,D); Z=Sub(E,Y) (for interleaved) @@ -194,8 +204,9 @@ TEST_F(MklLayoutPassTest, NodeMerge_Conv2DWithBias_Positive1) { " input: ['E', 'Y']}"); EXPECT_EQ(DoMklLayoutOptimizationPass(), "A(Input);B(Input);D(Input);DMT/_0(Const);E(_MklConv2DWithBias);" - "M(_MklInput2);N(_MklInput2);Y(Input);Z(Sub)|A->E;B->E:1;D->E:2;" - "DMT/_0->E:5;E->Z;M:1->E:3;N:1->E:4;Y->Z:1"); + "M(_MklInput2);N(_MklInput2);Y(Input);Z(Sub)|A->E;" + "A:control->DMT/_0:control;B->E:1;D->E:2;DMT/_0->E:5;E->Z;" + "M:1->E:3;N:1->E:4;Y->Z:1"); } // C=Conv2D(A,B); E=BiasAdd(C,D); Z=Sub(E,Y); @@ -226,8 +237,9 @@ TEST_F(MklLayoutPassTest, NodeMerge_Conv2DWithBias_Positive2) { EXPECT_EQ(DoMklLayoutOptimizationPass(), "A(Input);B(Input);D(Input);DMT/_0(Const);DMT/_1(Const);" "DMT/_2(Const);E(_MklConv2DWithBias);Y(Input);Z(Sub)|" - "A->E;B->E:1;D->E:2;DMT/_0->E:3;DMT/_1->E:4;DMT/_2->E:5;" - "E->Z;Y->Z:1"); + "A->E;A:control->DMT/_0:control;A:control->DMT/_1:control;" + "A:control->DMT/_2:control;B->E:1;D->E:2;DMT/_0->E:3;DMT/_1->E:4;" + "DMT/_2->E:5;E->Z;Y->Z:1"); } // Graph contains only _MklConv2D, no AddBias. @@ -330,9 +342,6 @@ TEST_F(MklLayoutPassTest, NodeMerge_Conv2DWithBias_Negative_AttrMismatch) { "N(_MklInput)|A->C;B->C:1;C->E;D->E:1;M->C:2;N->C:3"); } -// Disabling Conv2DBackpropBias test for now as we have disabled rewrite -// of BiasAddGrad into BackpropBias -#if 0 // Test set 2: _MklConv2D..BiasAddGrad -> _MklConv2DWithBiasBackpropBias // rewrite tests @@ -361,18 +370,17 @@ TEST_F(MklLayoutPassTest, NodeMerge_Conv2DBackprop_Positive) { " input: ['E'] }"); EXPECT_EQ(DoMklLayoutOptimizationPass(), "A(Input);B(Input);C(Input);D(_MklConv2DWithBias);DMT/_0(Const);" - "E(Sub);F(_MklConv2DWithBiasBackpropBias);M(_MklInput);N(_MklInput);" - "O(_MklInput)|A->D;A->E:1;B->D:1;C->D:2;D->E;DMT/_0->F:1;E->F;" - "M->D:3;N->D:4;O->D:5"); + "E(Sub);F(_MklConv2DWithBiasBackpropBias);M(_MklInput);" + "N(_MklInput);O(_MklInput)|A->D;A->E:1;B->D:1;C->D:2;D->E;" + "DMT/_0->F:1;E->F;E:control->DMT/_0:control;M->D:3;N->D:4;" + "O->D:5"); } -#endif -// No _MklConv2D in context, but Conv2D in context. -// Only Conv2D would be rewritten to _MklConv2D, but no rewrite -// for BiasAddGrad should happen. +// No _MklConv2DWithBias in context, but _MklConv2D in context. +// No rewrite for BiasAddGrad should happen. // C=_MklConv2D(A,M,B,N); D=Sub(C,A); E=BiasAddGrad(D) (for interleaved) // C=_MklConv2D(A,B,M,N); D=Sub(C,A); E=BiasAddGrad(D) (for contiguous) -TEST_F(MklLayoutPassTest, NodeMerge_Conv2DBackprop_Neg_No_MklConv2DWithBias) { +TEST_F(MklLayoutPassTest, NodeMerge_Conv2DBackprop_Neg_NoMklConv2DWithBias) { InitGraph( "node { name: 'A' op: 'Input'}" "node { name: 'B' op: 'Input'}" @@ -507,8 +515,10 @@ TEST_F(MklLayoutPassTest, NodeRewrite_Conv2D_Basic) { "node { name: 'D' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" " input: ['B', 'C'] }"); EXPECT_EQ(DoMklLayoutOptimizationPass(), - "A(Input);B(Input);C(_MklConv2D);D(Mul);DMT/_0(Const);DMT/_1(Const)|" - "A->C;B->C:1;B->D;C->D:1;DMT/_0->C:2;DMT/_1->C:3"); + "A(Input);B(Input);C(_MklConv2D);D(Mul);DMT/_0(Const);" + "DMT/_1(Const)|A->C;A:control->DMT/_0:control;" + "A:control->DMT/_1:control;B->C:1;B->D;C->D:1;DMT/_0->C:2;" + "DMT/_1->C:3"); } // 2 Conv2D Ops in sequence. Both should get transformed and 1st Conv2D will @@ -535,7 +545,9 @@ TEST_F(MklLayoutPassTest, NodeRewrite_Conv2D_Positive1) { " input: ['C', 'D'] }"); EXPECT_EQ(DoMklLayoutOptimizationPass(), "A(Input);B(Input);C(_MklConv2D);D(_MklConv2D);DMT/_0(Const);" - "DMT/_1(Const);DMT/_2(Const);E(Mul)|A->C;A->D;B->C:1;C->D:1;C->E;" + "DMT/_1(Const);DMT/_2(Const);E(Mul)|A->C;A->D;" + "A:control->DMT/_0:control;A:control->DMT/_1:control;" + "A:control->DMT/_2:control;B->C:1;C->D:1;C->E;" "C:1->D:3;D->E:1;DMT/_0->C:2;DMT/_1->C:3;DMT/_2->D:2"); } @@ -558,6 +570,50 @@ TEST_F(MklLayoutPassTest, NodeRewrite_Conv2D_Negative_UnsupportedType) { "A->C;B->C:1;B->D;C->D:1"); } +TEST_F(MklLayoutPassTest, NodeRewrite_Conv2DGradFilter_Positive) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'Int32Input'}" + "node { name: 'C' op: 'Input'}" + "node { name: 'D' op: 'Conv2DBackpropFilter'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " attr { key: 'use_cudnn_on_gpu' value { b: false } }" + " attr { key: 'strides' value { list: {i: 1, i:1, i:1, i:1} } }" + " attr { key: 'padding' value { s: 'SAME' } }" + " input: ['A', 'B', 'C']}" + "node { name: 'E' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'D'] }"); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(Int32Input);C(Input);D(_MklConv2DBackpropFilter);" + "DMT/_0(Const);DMT/_1(Const);DMT/_2(Const);E(Mul)|" + "A->D;A->E;A:control->DMT/_0:control;A:control->DMT/_1:control;" + "A:control->DMT/_2:control;B->D:1;C->D:2;D->E:1;DMT/_0->D:3;" + "DMT/_1->D:4;DMT/_2->D:5"); +} + +TEST_F(MklLayoutPassTest, NodeRewrite_Conv2DGradInput_Positive) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'Int32Input'}" + "node { name: 'C' op: 'Input'}" + "node { name: 'D' op: 'Conv2DBackpropInput'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " attr { key: 'use_cudnn_on_gpu' value { b: false } }" + " attr { key: 'strides' value { list: {i: 1, i:1, i:1, i:1} } }" + " attr { key: 'padding' value { s: 'SAME' } }" + " input: ['B', 'A', 'C']}" + "node { name: 'E' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'D'] }"); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(Int32Input);C(Input);D(_MklConv2DBackpropInput);" + "DMT/_0(Const);DMT/_1(Const);DMT/_2(Const);E(Mul)|" + "A->D:1;A->E;B->D;B:control->DMT/_0:control;" + "B:control->DMT/_1:control;B:control->DMT/_2:control;C->D:2;" + "D->E:1;DMT/_0->D:3;DMT/_1->D:4;DMT/_2->D:5"); +} + // Concat Op test: Concat with no Mkl layer feeding it TEST_F(MklLayoutPassTest, NodeRewrite_Concat_Basic) { InitGraph( @@ -572,13 +628,14 @@ TEST_F(MklLayoutPassTest, NodeRewrite_Concat_Basic) { "node { name: 'D' op: 'Concat'" " attr { key: 'T' value { type: DT_FLOAT } }" " attr { key: 'N' value { i: 2 } }" - " input: ['A', 'B']}" + " input: ['A', 'B:0', 'B:1']}" "node { name: 'E' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" " input: ['C', 'D'] }"); EXPECT_EQ(DoMklLayoutOptimizationPass(), "A(Const);B(InputList);C(Input);D(_MklConcat);DMT/_0(Const);" - "DMT/_1(Const);DMT/_2(Const);E(Mul)|A->D;B->D:1;B->D:2;C->E;" - "D->E:1;DMT/_0->D:3;DMT/_1->D:4;DMT/_2->D:5"); + "DMT/_1(Const);DMT/_2(Const);E(Mul)|A->D;A:control->DMT/_0:control;" + "A:control->DMT/_1:control;A:control->DMT/_2:control;B->D:1;" + "B:1->D:2;C->E;D->E:1;DMT/_0->D:3;DMT/_1->D:4;DMT/_2->D:5"); } // Concat with 2 Mkl layers feeding it @@ -616,9 +673,12 @@ TEST_F(MklLayoutPassTest, NodeRewrite_Concat_Input_Mkl) { EXPECT_EQ(DoMklLayoutOptimizationPass(), "A(Input);B(Input);C(Input);D(Input);DMT/_0(Const);DMT/_1(Const);" "DMT/_2(Const);DMT/_3(Const);DMT/_4(Const);E(_MklConv2D);" - "F(_MklConv2D);G(Const);H(_MklConcat);I(Mul)|A->E;A->I;B->E:1;C->F;" + "F(_MklConv2D);G(Const);H(_MklConcat);I(Mul)|A->E;A->I;" + "A:control->DMT/_2:control;A:control->DMT/_3:control;" + "B->E:1;C->F;C:control->DMT/_0:control;C:control->DMT/_1:control;" "D->F:1;DMT/_0->F:2;DMT/_1->F:3;DMT/_2->E:2;DMT/_3->E:3;" - "DMT/_4->H:3;E->H:1;E:1->H:4;F->H:2;F:1->H:5;G->H;H->I:1"); + "DMT/_4->H:3;E->H:1;E:1->H:4;F->H:2;F:1->H:5;G->H;" + "G:control->DMT/_4:control;H->I:1"); } // Concat with 1 Mkl and 1 non-Mkl layer feeding it @@ -651,12 +711,12 @@ TEST_F(MklLayoutPassTest, NodeRewrite_Concat_Input_MixedMkl) { EXPECT_EQ(DoMklLayoutOptimizationPass(), "A(Input);B(Input);C(Input);D(Input);DMT/_0(Const);DMT/_1(Const);" "DMT/_2(Const);DMT/_3(Const);E(_MklConv2D);F(Mul);G(Const);" - "H(_MklConcat);I(Mul)|A->E;A->I;B->E:1;C->F;D->F:1;DMT/_0->E:2;" + "H(_MklConcat);I(Mul)|A->E;A->I;A:control->DMT/_0:control;" + "A:control->DMT/_1:control;B->E:1;C->F;D->F:1;DMT/_0->E:2;" "DMT/_1->E:3;DMT/_2->H:3;DMT/_3->H:5;E->H:1;E:1->H:4;F->H:2;" - "G->H;H->I:1"); + "G->H;G:control->DMT/_2:control;G:control->DMT/_3:control;H->I:1"); } -#if 0 // ConcatV2 Op test: ConcatV2 with no Mkl layer feeding it TEST_F(MklLayoutPassTest, NodeRewrite_ConcatV2_Basic) { InitGraph( @@ -676,11 +736,12 @@ TEST_F(MklLayoutPassTest, NodeRewrite_ConcatV2_Basic) { "node { name: 'E' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" " input: ['C', 'D'] }"); EXPECT_EQ(DoMklLayoutOptimizationPass(), - "A(Const);B(InputList);C(Input);D(_MklConcat);DMT/_0(Const);" - "DMT/_1(Const);DMT/_2(Const);E(Mul)|A->D:2;B->D;B:1->D:1;C->E;" - "D->E:1;DMT/_0->D:3;DMT/_1->D:4;DMT/_2->D:5"); + "A(Const);B(InputList);C(Input);D(_MklConcatV2);DMT/_0(Const);" + "DMT/_1(Const);DMT/_2(Const);E(Mul)|A->D:2;B->D;B:1->D:1;" + "B:control->DMT/_0:control;B:control->DMT/_1:control;" + "B:control->DMT/_2:control;C->E;D->E:1;DMT/_0->D:3;" + "DMT/_1->D:4;DMT/_2->D:5"); } -#endif // ConcatV2 with 2 Mkl layers feeding it TEST_F(MklLayoutPassTest, NodeRewrite_ConcatV2_Input_Mkl) { @@ -718,9 +779,12 @@ TEST_F(MklLayoutPassTest, NodeRewrite_ConcatV2_Input_Mkl) { EXPECT_EQ(DoMklLayoutOptimizationPass(), "A(Input);B(Input);C(Input);D(Input);DMT/_0(Const);DMT/_1(Const);" "DMT/_2(Const);DMT/_3(Const);DMT/_4(Const);E(_MklConv2D);" - "F(_MklConv2D);G(Const);H(_MklConcatV2);I(Mul)|A->E;A->I;B->E:1;C->F;" + "F(_MklConv2D);G(Const);H(_MklConcatV2);I(Mul)|A->E;A->I;" + "A:control->DMT/_2:control;A:control->DMT/_3:control;B->E:1;C->F;" + "C:control->DMT/_0:control;C:control->DMT/_1:control;" "D->F:1;DMT/_0->F:2;DMT/_1->F:3;DMT/_2->E:2;DMT/_3->E:3;" - "DMT/_4->H:5;E->H;E:1->H:3;F->H:1;F:1->H:4;G->H:2;H->I:1"); + "DMT/_4->H:5;E->H;E:1->H:3;E:control->DMT/_4:control;F->H:1;" + "F:1->H:4;G->H:2;H->I:1"); } // ConcatV2 with 1 Mkl and 1 non-Mkl layer feeding it @@ -754,11 +818,175 @@ TEST_F(MklLayoutPassTest, NodeRewrite_ConcatV2_Input_MixedMkl) { EXPECT_EQ(DoMklLayoutOptimizationPass(), "A(Input);B(Input);C(Input);D(Input);DMT/_0(Const);DMT/_1(Const);" "DMT/_2(Const);DMT/_3(Const);E(_MklConv2D);F(Mul);G(Const);" - "H(_MklConcatV2);I(Mul)|A->E;A->I;B->E:1;C->F;D->F:1;DMT/_0->E:2;" - "DMT/_1->E:3;DMT/_2->H:4;DMT/_3->H:5;E->H;E:1->H:3;F->H:1;" + "H(_MklConcatV2);I(Mul)|A->E;A->I;A:control->DMT/_0:control;" + "A:control->DMT/_1:control;B->E:1;C->F;D->F:1;DMT/_0->E:2;" + "DMT/_1->E:3;DMT/_2->H:4;DMT/_3->H:5;E->H;E:1->H:3;" + "E:control->DMT/_2:control;E:control->DMT/_3:control;F->H:1;" "G->H:2;H->I:1"); } +TEST_F(MklLayoutPassTest, NodeRewrite_Relu_Positive) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'Relu'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A'] }" + "node { name: 'C' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'B'] }"); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(_MklRelu);C(Mul);DMT/_0(Const)|A->B;A->C;" + "A:control->DMT/_0:control;B->C:1;DMT/_0->B:1"); +} + +TEST_F(MklLayoutPassTest, NodeRewrite_ReluGrad_Positive) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'Input'}" + "node { name: 'C' op: 'ReluGrad'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'B'] }" + "node { name: 'D' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'C'] }"); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(Input);C(_MklReluGrad);D(Mul);DMT/_0(Const);" + "DMT/_1(Const)|A->C;A->D;A:control->DMT/_0:control;" + "A:control->DMT/_1:control;B->C:1;C->D:1;DMT/_0->C:2;DMT/_1->C:3"); +} + +TEST_F(MklLayoutPassTest, NodeRewrite_ReluReluGrad_Positive) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'Relu'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A'] }" + "node { name: 'C' op: 'ReluGrad'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'B'] }" + "node { name: 'D' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'C'] }"); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(_MklRelu);C(_MklReluGrad);D(Mul);DMT/_0(Const);" + "DMT/_1(Const)|A->B;A->C;A->D;A:control->DMT/_0:control;" + "A:control->DMT/_1:control;B->C:1;B:1->C:3;C->D:1;DMT/_0->B:1;" + "DMT/_1->C:2"); +} + +TEST_F(MklLayoutPassTest, NodeRewrite_AvgPool_Positive) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'AvgPool'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " attr { key: 'ksize' value { list: {i: 1, i:1, i:3, i:3} } }" + " attr { key: 'padding' value { s: 'VALID' } }" + " attr { key: 'strides' value { list: {i: 1, i:1, i:2, i:2} } }" + " input: ['A'] }" + "node { name: 'C' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'B'] }"); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(_MklAvgPool);C(Mul);DMT/_0(Const)|A->B;A->C;" + "A:control->DMT/_0:control;B->C:1;DMT/_0->B:1"); +} + +TEST_F(MklLayoutPassTest, NodeRewrite_AvgPoolGrad_Positive) { + InitGraph( + "node { name: 'A' op: 'Int32Input'}" + "node { name: 'B' op: 'Input'}" + "node { name: 'C' op: 'AvgPoolGrad' " + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " attr { key: 'ksize' value { list: {i: 1, i:1, i:3, i:3} } }" + " attr { key: 'padding' value { s: 'VALID' } }" + " attr { key: 'strides' value { list: {i: 1, i:1, i:2, i:2} } }" + " input: ['A', 'B'] }" + "node { name: 'D' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['B', 'C'] }"); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Int32Input);B(Input);C(_MklAvgPoolGrad);D(Mul);DMT/_0(Const);" + "DMT/_1(Const)|A->C;A:control->DMT/_0:control;" + "A:control->DMT/_1:control;B->C:1;B->D;C->D:1;DMT/_0->C:2;" + "DMT/_1->C:3"); +} + +TEST_F(MklLayoutPassTest, NodeRewrite_AvgPoolAvgPoolGrad_Positive) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'I' op: 'Int32Input'}" + "node { name: 'B' op: 'AvgPool'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " attr { key: 'ksize' value { list: {i: 1, i:1, i:3, i:3} } }" + " attr { key: 'padding' value { s: 'VALID' } }" + " attr { key: 'strides' value { list: {i: 1, i:1, i:2, i:2} } }" + " input: ['A'] }" + "node { name: 'C' op: 'AvgPoolGrad' " + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " attr { key: 'ksize' value { list: {i: 1, i:1, i:3, i:3} } }" + " attr { key: 'padding' value { s: 'VALID' } }" + " attr { key: 'strides' value { list: {i: 1, i:1, i:2, i:2} } }" + " input: ['I', 'B'] }" + "node { name: 'D' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'C'] }"); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(_MklAvgPool);C(_MklAvgPoolGrad);D(Mul);DMT/_0(Const);" + "DMT/_1(Const);I(Int32Input)|A->B;A->D;A:control->DMT/_0:control;" + "B->C:1;B:1->C:3;C->D:1;DMT/_0->B:1;DMT/_1->C:2;I->C;" + "I:control->DMT/_1:control"); +} + +TEST_F(MklLayoutPassTest, NodeRewrite_FusedBatchNormGrad_Positive) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'Input'}" + "node { name: 'C' op: 'Input'}" + "node { name: 'D' op: 'Input'}" + "node { name: 'E' op: 'Input'}" + "node { name: 'F' op: 'FusedBatchNormGrad'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " attr { key: 'epsilon' value { f: 0.0001 } }" + " attr { key: 'is_training' value { b: true } }" + " input: ['A', 'B', 'C', 'D', 'E'] }" + "node { name: 'G' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'F'] }"); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(Input);C(Input);D(Input);DMT/_0(Const);DMT/_1(Const);" + "DMT/_2(Const);DMT/_3(Const);DMT/_4(Const);E(Input);" + "F(_MklFusedBatchNormGrad);G(Mul)|A->F;A->G;" + "A:control->DMT/_0:control;A:control->DMT/_1:control;" + "A:control->DMT/_2:control;A:control->DMT/_3:control;" + "A:control->DMT/_4:control;B->F:1;C->F:2;D->F:3;" + "DMT/_0->F:5;DMT/_1->F:6;DMT/_2->F:7;DMT/_3->F:8;DMT/_4->F:9;" + "E->F:4;F->G:1"); +} + +TEST_F(MklLayoutPassTest, NodeRewrite_FusedBatchNorm_Positive) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'Input'}" + "node { name: 'C' op: 'Input'}" + "node { name: 'D' op: 'Input'}" + "node { name: 'E' op: 'Input'}" + "node { name: 'F' op: 'FusedBatchNorm'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " attr { key: 'epsilon' value { f: 0.0001 } }" + " attr { key: 'is_training' value { b: true } }" + " input: ['A', 'B', 'C', 'D', 'E'] }" + "node { name: 'G' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'F'] }"); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(Input);C(Input);D(Input);DMT/_0(Const);DMT/_1(Const);" + "DMT/_2(Const);DMT/_3(Const);DMT/_4(Const);E(Input);" + "F(_MklFusedBatchNorm);G(Mul)|A->F;A->G;" + "A:control->DMT/_0:control;A:control->DMT/_1:control;" + "A:control->DMT/_2:control;A:control->DMT/_3:control;" + "A:control->DMT/_4:control;B->F:1;C->F:2;D->F:3;" + "DMT/_0->F:5;DMT/_1->F:6;DMT/_2->F:7;DMT/_3->F:8;DMT/_4->F:9;" + "E->F:4;F->G:1"); +} + ///////////////////////////////////////////////////////////////////// // Unit tests related to rewriting node for workspace edges ///////////////////////////////////////////////////////////////////// @@ -802,13 +1030,13 @@ TEST_F(MklLayoutPassTest, MaxPoolLRN_Positive) { "node { name: 'H' op: 'Input'}" "node { name: 'I' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" " input: ['H', 'G'] }"); - EXPECT_EQ( - DoMklLayoutOptimizationPass(), + EXPECT_EQ(DoMklLayoutOptimizationPass(), "A(Input);B(_MklLRN);C(_MklMaxPool);D(Input);DMT/_0(Const);DMT/_1(Const);" - "DMT/_2(Const);E(_MklMaxPoolGrad);F(Input);G(_MklLRNGrad);H(Input);I(Mul)|" - "A->B;B->C;B->E;B->G:2;B:1->G:3;B:2->C:1;B:2->E:4;B:2->G:6;B:3->G:7;" - "C->E:1;C:1->E:3;C:2->E:5;C:3->E:7;D->E:2;DMT/_0->B:1;DMT/_1->E:6;" - "DMT/_2->G:5;E->G;E:1->G:4;F->G:1;G->I:1;H->I"); + "DMT/_2(Const);E(_MklMaxPoolGrad);F(Input);G(_MklLRNGrad);H(Input);" + "I(Mul)|A->B;A:control->DMT/_0:control;B->C;B->E;B->G:2;B:1->G:3;" + "B:2->C:1;B:2->E:4;B:2->G:6;B:3->G:7;B:control->DMT/_1:control;C->E:1;" + "C:1->E:3;C:2->E:5;C:3->E:7;D->E:2;DMT/_0->B:1;DMT/_1->E:6;DMT/_2->G:5;" + "E->G;E:1->G:4;E:control->DMT/_2:control;F->G:1;G->I:1;H->I"); } /* Test LRN->LRNGrad replacement by workspace nodes. */ @@ -838,8 +1066,9 @@ TEST_F(MklLayoutPassTest, LRN_Positive) { EXPECT_EQ(DoMklLayoutOptimizationPass(), "A(Input);B(_MklLRN);C(Input);D(Input);DMT/_0(Const);DMT/_1(Const);" "DMT/_2(Const);E(_MklLRNGrad);F(Mul)|" - "A->B;B->E:2;B:1->E:3;B:2->E:6;B:3->E:7;C->E;C->F;D->E:1;" - "DMT/_0->B:1;DMT/_1->E:4;DMT/_2->E:5;E->F:1"); + "A->B;A:control->DMT/_0:control;B->E:2;B:1->E:3;B:2->E:6;B:3->E:7;" + "C->E;C->F;C:control->DMT/_1:control;C:control->DMT/_2:control;" + "D->E:1;DMT/_0->B:1;DMT/_1->E:4;DMT/_2->E:5;E->F:1"); } /* Test LRN->LRNGrad replacement when only one of them is present. */ @@ -858,7 +1087,7 @@ TEST_F(MklLayoutPassTest, LRN_Negative1) { " input: ['A', 'B'] }"); EXPECT_EQ(DoMklLayoutOptimizationPass(), "A(Input);B(_MklLRN);C(Mul);DMT/_0(Const)|" - "A->B;A->C;B->C:1;DMT/_0->B:1"); + "A->B;A->C;A:control->DMT/_0:control;B->C:1;DMT/_0->B:1"); } /* Test LRN->LRNGrad replacement when only one of them is present. */ @@ -880,8 +1109,10 @@ TEST_F(MklLayoutPassTest, LRN_Negative2) { EXPECT_EQ(DoMklLayoutOptimizationPass(), "A(Input);B(Input);C(Input);D(_MklLRNGrad);DMT/_0(Const);" "DMT/_1(Const);DMT/_2(Const);DMT/_3(Const);DMT/_4(Const);E(Mul)|" - "A->D;A->E;B->D:1;C->D:2;D->E:1;DMT/_0->D:3;DMT/_1->D:7;" - "DMT/_2->D:4;DMT/_3->D:5;DMT/_4->D:6"); + "A->D;A->E;A:control->DMT/_0:control;A:control->DMT/_1:control;" + "A:control->DMT/_2:control;A:control->DMT/_3:control;" + "A:control->DMT/_4:control;B->D:1;C->D:2;D->E:1;DMT/_0->D:3;" + "DMT/_1->D:7;DMT/_2->D:4;DMT/_3->D:5;DMT/_4->D:6"); } /* Test LRN->LRNGrad negative case, where single LRN feeds @@ -920,9 +1151,13 @@ TEST_F(MklLayoutPassTest, LRN_Negative3) { EXPECT_EQ(DoMklLayoutOptimizationPass(), "A(Input);B(_MklLRN);C(Input);D(Input);DMT/_0(Const);DMT/_1(Const);" "DMT/_2(Const);DMT/_3(Const);DMT/_4(Const);DMT/_5(Const);" - "DMT/_6(Const);E(_MklLRNGrad);F(_MklLRNGrad);G(Mul)|A->B;B->E:2;" - "B->F:1;B:1->E:3;B:2->E:6;B:2->F:5;B:3->E:7;C->E;C->F;D->E:1;" - "D->F:2;DMT/_0->B:1;DMT/_1->F:3;DMT/_2->F:7;DMT/_3->F:4;" + "DMT/_6(Const);E(_MklLRNGrad);F(_MklLRNGrad);G(Mul)|A->B;" + "A:control->DMT/_0:control;B->E:2;" + "B->F:1;B:1->E:3;B:2->E:6;B:2->F:5;B:3->E:7;C->E;C->F;" + "C:control->DMT/_1:control;C:control->DMT/_2:control;" + "C:control->DMT/_3:control;C:control->DMT/_4:control;" + "C:control->DMT/_5:control;C:control->DMT/_6:control;" + "D->E:1;D->F:2;DMT/_0->B:1;DMT/_1->F:3;DMT/_2->F:7;DMT/_3->F:4;" "DMT/_4->F:6;DMT/_5->E:4;DMT/_6->E:5;E->G;F->G:1"); } @@ -951,8 +1186,9 @@ TEST_F(MklLayoutPassTest, NodeWorkspace_MaxPool_Positive) { EXPECT_EQ(DoMklLayoutOptimizationPass(), "A(Input);B(_MklMaxPool);C(Input);D(Input);DMT/_0(Const);" "DMT/_1(Const);DMT/_2(Const);E(_MklMaxPoolGrad);F(Mul)|" - "A->B;B->E:1;B:1->E:3;B:2->E:5;B:3->E:7;C->E;C->F;D->E:2;" - "DMT/_0->B:1;DMT/_1->E:4;DMT/_2->E:6;E->F:1"); + "A->B;A:control->DMT/_0:control;B->E:1;B:1->E:3;B:2->E:5;B:3->E:7;" + "C->E;C->F;C:control->DMT/_1:control;C:control->DMT/_2:control;" + "D->E:2;DMT/_0->B:1;DMT/_1->E:4;DMT/_2->E:6;E->F:1"); } // Test MaxPool>MaxPoolGrad replacement when only one of them is present. @@ -972,7 +1208,7 @@ TEST_F(MklLayoutPassTest, NodeWorkspace_MaxPool_Negative1) { " input: ['A', 'B'] }"); EXPECT_EQ(DoMklLayoutOptimizationPass(), "A(Input);B(_MklMaxPool);C(Mul);DMT/_0(Const)|" - "A->B;A->C;B->C:1;DMT/_0->B:1"); + "A->B;A->C;A:control->DMT/_0:control;B->C:1;DMT/_0->B:1"); } // Test MaxPoolGrad replacement when only one of them is present. @@ -995,8 +1231,374 @@ TEST_F(MklLayoutPassTest, NodeWorkspace_MaxPool_Negative2) { EXPECT_EQ(DoMklLayoutOptimizationPass(), "A(Input);B(Input);C(Input);D(_MklMaxPoolGrad);DMT/_0(Const);" "DMT/_1(Const);DMT/_2(Const);DMT/_3(Const);DMT/_4(Const);E(Mul)|" - "A->D;A->E;B->D:1;C->D:2;D->E:1;DMT/_0->D:3;DMT/_1->D:7;" - "DMT/_2->D:4;DMT/_3->D:5;DMT/_4->D:6"); + "A->D;A->E;A:control->DMT/_0:control;A:control->DMT/_1:control;" + "A:control->DMT/_2:control;A:control->DMT/_3:control;" + "A:control->DMT/_4:control;B->D:1;C->D:2;D->E:1;DMT/_0->D:3;" + "DMT/_1->D:7;DMT/_2->D:4;DMT/_3->D:5;DMT/_4->D:6"); +} + +// Test MaxPool handling for batch-wise pooling (NCHW) +// No rewrite should take place in such case +TEST_F(MklLayoutPassTest, NodeWorkspace_MaxPool_Negative3) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'MaxPool'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " attr { key: 'ksize' value { list: {i: 2, i:1, i:1, i:1} } }" + " attr { key: 'padding' value { s: 'VALID' } }" + " attr { key: 'strides' value { list: {i: 1, i:1, i:1, i:1} } }" + " input: ['A'] }" + "node { name: 'C' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'B'] }"); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(MaxPool);C(Mul)|A->B;A->C;B->C:1"); +} + +// Test MaxPool handling for batch-wise pooling (NCHW) +// No rewrite should take place in such case +TEST_F(MklLayoutPassTest, NodeWorkspace_MaxPool_Negative4) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'MaxPool'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " attr { key: 'ksize' value { list: {i: 1, i:1, i:1, i:1} } }" + " attr { key: 'padding' value { s: 'VALID' } }" + " attr { key: 'strides' value { list: {i: 2, i:1, i:1, i:1} } }" + " input: ['A'] }" + "node { name: 'C' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'B'] }"); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(MaxPool);C(Mul)|A->B;A->C;B->C:1"); +} + +// Test MaxPool handling for depth-wise pooling (NHWC) +// No rewrite should take place in such case +TEST_F(MklLayoutPassTest, NodeWorkspace_MaxPool_Negative5) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'MaxPool'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " attr { key: 'ksize' value { list: {i: 1, i:2, i:1, i:1} } }" + " attr { key: 'padding' value { s: 'VALID' } }" + " attr { key: 'strides' value { list: {i: 1, i:1, i:1, i:1} } }" + " input: ['A'] }" + "node { name: 'C' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'B'] }"); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(MaxPool);C(Mul)|A->B;A->C;B->C:1"); +} + +// Test MaxPool handling for depth-wise pooling (NCHW) +// No rewrite should take place in such case +TEST_F(MklLayoutPassTest, NodeWorkspace_MaxPool_Negative6) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'MaxPool'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " attr { key: 'ksize' value { list: {i: 1, i:1, i:1, i:1} } }" + " attr { key: 'padding' value { s: 'VALID' } }" + " attr { key: 'strides' value { list: {i: 1, i:2, i:1, i:1} } }" + " input: ['A'] }" + "node { name: 'C' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'B'] }"); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(MaxPool);C(Mul)|A->B;A->C;B->C:1"); +} + +// Test MaxPool handling for batch-wise pooling (NHWC) +// No rewrite should take place in such case +TEST_F(MklLayoutPassTest, NodeWorkspace_MaxPool_Negative7) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'MaxPool'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NHWC' } }" + " attr { key: 'ksize' value { list: {i: 2, i:1, i:1, i:1} } }" + " attr { key: 'padding' value { s: 'VALID' } }" + " attr { key: 'strides' value { list: {i: 1, i:1, i:1, i:1} } }" + " input: ['A'] }" + "node { name: 'C' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'B'] }"); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(MaxPool);C(Mul)|A->B;A->C;B->C:1"); +} + +// Test MaxPool handling for batch-wise pooling (NHWC) +// No rewrite should take place in such case +TEST_F(MklLayoutPassTest, NodeWorkspace_MaxPool_Negative8) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'MaxPool'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NHWC' } }" + " attr { key: 'ksize' value { list: {i: 1, i:1, i:1, i:1} } }" + " attr { key: 'padding' value { s: 'VALID' } }" + " attr { key: 'strides' value { list: {i: 2, i:1, i:1, i:1} } }" + " input: ['A'] }" + "node { name: 'C' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'B'] }"); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(MaxPool);C(Mul)|A->B;A->C;B->C:1"); +} + +// Test MaxPool handling for depth-wise pooling (NHWC) +// No rewrite should take place in such case +TEST_F(MklLayoutPassTest, NodeWorkspace_MaxPool_Negative9) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'MaxPool'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NHWC' } }" + " attr { key: 'ksize' value { list: {i: 1, i:1, i:1, i:2} } }" + " attr { key: 'padding' value { s: 'VALID' } }" + " attr { key: 'strides' value { list: {i: 1, i:1, i:1, i:1} } }" + " input: ['A'] }" + "node { name: 'C' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'B'] }"); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(MaxPool);C(Mul)|A->B;A->C;B->C:1"); +} + +// Test MaxPool handling for depth-wise pooling (NHWC) +// No rewrite should take place in such case +TEST_F(MklLayoutPassTest, NodeWorkspace_MaxPool_Negative10) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'MaxPool'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NHWC' } }" + " attr { key: 'ksize' value { list: {i: 1, i:1, i:1, i:1} } }" + " attr { key: 'padding' value { s: 'VALID' } }" + " attr { key: 'strides' value { list: {i: 1, i:1, i:1, i:2} } }" + " input: ['A'] }" + "node { name: 'C' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'B'] }"); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(MaxPool);C(Mul)|A->B;A->C;B->C:1"); +} + +///////////////////////////////////////////////////////////////////// + +// Single Conv2D Op on GPU device +// No rewrite should happen +TEST_F(MklLayoutPassTest, NodeRewrite_Conv2D_DeviceTest) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'Input'}" + "node { name: 'C' op: 'Conv2D'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " attr { key: 'use_cudnn_on_gpu' value { b: false } }" + " attr { key: 'strides' value { list: {i: 1, i:1, i:1, i:1} } }" + " attr { key: 'padding' value { s: 'SAME' } }" + " input: ['A', 'B']}" + "node { name: 'D' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['B', 'C'] }", kGPUDevice); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(Input);C(Conv2D);D(Mul)|A->C;B->C:1;B->D;C->D:1"); +} + +TEST_F(MklLayoutPassTest, NodeMerge_Conv2DBackprop_DeviceTest) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'Input'}" + "node { name: 'C' op: 'Input'}" + "node { name: 'M' op: '_MklInput'}" + "node { name: 'N' op: '_MklInput'}" + "node { name: 'O' op: '_MklInput'}" + "node { name: 'D' op: '_MklConv2DWithBias'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " attr { key: 'use_cudnn_on_gpu' value { b: false } }" + " attr { key: 'strides' value { list: {i: 1, i:1, i:1, i:1} } }" + " attr { key: 'padding' value { s: 'SAME' } }" + " input: ['A', 'B', 'C', 'M', 'N', 'O']}" + "node { name: 'E' op: 'Sub'" + " attr {key: 'T' value { type: DT_FLOAT } }" + " input: ['D', 'A']}" + "node { name: 'F' op: 'BiasAddGrad'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " input: ['E'] }", kGPUDevice); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(Input);C(Input);D(_MklConv2DWithBias);" + "E(Sub);F(BiasAddGrad);M(_MklInput);N(_MklInput);" + "O(_MklInput)|A->D;A->E:1;B->D:1;C->D:2;D->E;E->F;" + "M->D:3;N->D:4;O->D:5"); +} + +TEST_F(MklLayoutPassTest, NodeRewrite_Conv2DGradFilter_DeviceTest) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'Int32Input'}" + "node { name: 'C' op: 'Input'}" + "node { name: 'D' op: 'Conv2DBackpropFilter'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " attr { key: 'use_cudnn_on_gpu' value { b: false } }" + " attr { key: 'strides' value { list: {i: 1, i:1, i:1, i:1} } }" + " attr { key: 'padding' value { s: 'SAME' } }" + " input: ['A', 'B', 'C']}" + "node { name: 'E' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'D'] }", kGPUDevice); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(Int32Input);C(Input);D(Conv2DBackpropFilter);E(Mul)|" + "A->D;A->E;B->D:1;C->D:2;D->E:1"); +} + +TEST_F(MklLayoutPassTest, NodeRewrite_Relu_DeviceTest) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'Relu'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A'] }" + "node { name: 'C' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'B'] }", kGPUDevice); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(Relu);C(Mul)|A->B;A->C;B->C:1"); +} + +TEST_F(MklLayoutPassTest, NodeRewrite_ReluGrad_DeviceTest) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'Input'}" + "node { name: 'C' op: 'ReluGrad'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'B'] }" + "node { name: 'D' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'C'] }", kGPUDevice); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(Input);C(ReluGrad);D(Mul)|A->C;A->D;B->C:1;C->D:1"); +} + +TEST_F(MklLayoutPassTest, NodeRewrite_MaxPool_DeviceTest) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'MaxPool'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NHWC' } }" + " attr { key: 'ksize' value { list: {i: 1, i:1, i:1, i:1} } }" + " attr { key: 'padding' value { s: 'VALID' } }" + " attr { key: 'strides' value { list: {i: 1, i:1, i:1, i:1} } }" + " input: ['A'] }" + "node { name: 'C' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'B'] }", kGPUDevice); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(MaxPool);C(Mul)|A->B;A->C;B->C:1"); +} + +TEST_F(MklLayoutPassTest, NodeRewrite_AvgPool_DeviceTest) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'AvgPool'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NHWC' } }" + " attr { key: 'ksize' value { list: {i: 1, i:1, i:1, i:1} } }" + " attr { key: 'padding' value { s: 'VALID' } }" + " attr { key: 'strides' value { list: {i: 1, i:1, i:1, i:1} } }" + " input: ['A'] }" + "node { name: 'C' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'B'] }", kGPUDevice); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(AvgPool);C(Mul)|A->B;A->C;B->C:1"); +} + +// Concat Op test: Concat with no Mkl layer feeding it +TEST_F(MklLayoutPassTest, NodeRewrite_Concat_DeviceTest) { + InitGraph( + "node { name: 'A' op: 'Const' " + " attr { key: 'dtype' value { type: DT_INT32 } }" + " attr { key: 'value' value { " + " tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } " + " int_val: 0 } } } }" + "node { name: 'B' op: 'InputList'" + " attr { key: 'N' value { i: 2 } }}" + "node { name: 'C' op: 'Input'}" + "node { name: 'D' op: 'Concat'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'N' value { i: 2 } }" + " input: ['A', 'B:0', 'B:1']}" + "node { name: 'E' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['C', 'D'] }", kGPUDevice); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Const);B(InputList);C(Input);D(Concat);E(Mul)|A->D;" + "B->D:1;B:1->D:2;C->E;D->E:1"); +} + +TEST_F(MklLayoutPassTest, NodeRewrite_ConcatV2_DeviceTest) { + InitGraph( + "node { name: 'A' op: 'Const' " + " attr { key: 'dtype' value { type: DT_INT32 } }" + " attr { key: 'value' value { " + " tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } " + " int_val: 0 } } } }" + "node { name: 'B' op: 'InputList'" + " attr { key: 'N' value { i: 2 } }}" + "node { name: 'C' op: 'Input'}" + "node { name: 'D' op: 'ConcatV2'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'Tidx' value { type: DT_INT32 } }" + " attr { key: 'N' value { i: 2 } }" + " input: ['B:0', 'B:1', 'A']}" + "node { name: 'E' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['C', 'D'] }", kGPUDevice); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Const);B(InputList);C(Input);D(ConcatV2);E(Mul)|" + "A->D:2;B->D;B:1->D:1;C->E;D->E:1"); +} + +TEST_F(MklLayoutPassTest, NodeRewrite_FusedBatchNorm_DeviceTest) { + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'Input'}" + "node { name: 'C' op: 'Input'}" + "node { name: 'D' op: 'Input'}" + "node { name: 'E' op: 'Input'}" + "node { name: 'F' op: 'FusedBatchNorm'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " attr { key: 'epsilon' value { f: 0.0001 } }" + " attr { key: 'is_training' value { b: true } }" + " input: ['A', 'B', 'C', 'D', 'E'] }" + "node { name: 'G' op: 'Mul' attr { key: 'T' value { type: DT_FLOAT } }" + " input: ['A', 'F'] }", kGPUDevice); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(Input);C(Input);D(Input);E(Input);" + "F(FusedBatchNorm);G(Mul)|A->F;A->G;B->F:1;C->F:2;D->F:3;" + "E->F:4;F->G:1"); +} + +TEST_F(MklLayoutPassTest, NodeMerge_Conv2DWithBias_DeviceTest) { + CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS); + InitGraph( + "node { name: 'A' op: 'Input'}" + "node { name: 'B' op: 'Input'}" + "node { name: 'M' op: '_MklInput'}" + "node { name: 'N' op: '_MklInput'}" + "node { name: 'C' op: '_MklConv2D'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " attr { key: 'use_cudnn_on_gpu' value { b: false } }" + " attr { key: 'strides' value { list: {i: 1, i:1, i:1, i:1} } }" + " attr { key: 'padding' value { s: 'SAME' } }" + " input: ['A', 'B', 'M', 'N']}" + "node { name: 'D' op: 'Input'}" + "node { name: 'E' op: 'BiasAdd'" + " attr { key: 'T' value { type: DT_FLOAT } }" + " attr { key: 'data_format' value { s: 'NCHW' } }" + " input: ['C', 'D'] }" + "node { name: 'Y' op: 'Input'}" + "node { name: 'Z' op: 'Sub'" + " attr {key: 'T' value { type: DT_FLOAT } }" + " input: ['E', 'Y']}", kGPUDevice); + EXPECT_EQ(DoMklLayoutOptimizationPass(), + "A(Input);B(Input);C(_MklConv2D);D(Input);E(BiasAdd);" + "M(_MklInput);N(_MklInput);Y(Input);Z(Sub)|A->C;" + "B->C:1;C->E;D->E:1;E->Z;M->C:2;N->C:3;Y->Z:1"); } ///////////////////////////////////////////////////////////////////// diff --git a/tensorflow/core/graph/mkl_tfconversion_pass.cc b/tensorflow/core/graph/mkl_tfconversion_pass.cc index 55c280719c..590b3d030f 100644 --- a/tensorflow/core/graph/mkl_tfconversion_pass.cc +++ b/tensorflow/core/graph/mkl_tfconversion_pass.cc @@ -98,12 +98,13 @@ class MklToTfConversionPass : public GraphOptimizationPass { Status InsertConversionNodeOnEdge(std::unique_ptr<Graph>* g, Edge*); }; -// We register MklToTf insertion for phase 1 in post-partition grouping. -// We register this pass after partitioning so that we get a complete -// picture of inputs and outputs of the nodes in the graphs. +// We register MklToTf insertion for phase 2 in post-partition grouping +// because we register MklLayoutRewritePass for phase 1 in post-partition +// grouping. We register this pass after partitioning so that we get a +// complete picture of inputs and outputs of the nodes in the graphs. const OptimizationPassRegistry::Grouping kMklTfConvPassGroup = OptimizationPassRegistry::POST_PARTITIONING; -REGISTER_OPTIMIZATION(kMklTfConvPassGroup, 1, MklToTfConversionPass); +REGISTER_OPTIMIZATION(kMklTfConvPassGroup, 2, MklToTfConversionPass); Status MklToTfConversionPass::InsertConversionNodeOnEdge( std::unique_ptr<Graph>* g, Edge* e) { @@ -121,10 +122,12 @@ Status MklToTfConversionPass::InsertConversionNodeOnEdge( string data_format; TF_CHECK_OK(GetNodeAttr(src->def(), "T", &src_datatype)); - TF_CHECK_OK(GetNodeAttr(dst->def(), "T", &dst_datatype)); - if (src_datatype != dst_datatype) { - string err_msg = "T attribute of " + src->name() + " and " + dst->name() + - " do not match. Will not insert" + + bool dst_dtype_found = GetNodeAttr(dst->def(), "T", &dst_datatype) == + Status::OK(); + // We compare source and destination datatypes only when both are found. + if (dst_dtype_found && (src_datatype != dst_datatype)) { + string err_msg = "T attribute of " + src->name() + " and " + + dst->name() + " do not match. Will not insert" + " MklToTf node in such case."; return Status(error::Code::INVALID_ARGUMENT, err_msg.c_str()); } @@ -202,18 +205,19 @@ bool MklToTfConversionPass::RunPass(std::unique_ptr<Graph>* g) { << src->type_string() << " and " << dst->type_string(); // Let's get source and destination data type. - DataType src_datatype = DT_INVALID; - if (GetNodeAttr(src->def(), "T", &src_datatype) != Status::OK()) { - continue; - } // We cannot check datatype on destination node because destination node // may not be Mkl node. - DataType dst_datatype = DT_INVALID; - GetNodeAttr(dst->def(), "T", &dst_datatype); + DataType src_datatype; + DataType dst_datatype; + bool src_is_mkl_op = (GetNodeAttr(src->def(), "T", &src_datatype) == + Status::OK() && + IsMklSupportedOp(src->type_string(), src_datatype)); + bool dst_is_mkl_op = (GetNodeAttr(dst->def(), "T", &dst_datatype) == + Status::OK() && + IsMklSupportedOp(dst->type_string(), dst_datatype)); // Check if src with is Mkl-compliant, while dst is not Mkl-compliant. - if (IsMklSupportedOp(src->type_string(), src_datatype) && - !IsMklSupportedOp(dst->type_string(), dst_datatype)) { + if (src_is_mkl_op && !dst_is_mkl_op) { VLOG(1) << "MklToTfConversionPass: Scheduled nodes " << src->name() << " and " << dst->name() << " for inserting conversion nodes"; candidate_edges.push_back(const_cast<Edge*>(e)); diff --git a/tensorflow/core/graph/mkl_tfconversion_pass_test.cc b/tensorflow/core/graph/mkl_tfconversion_pass_test.cc index bd2cb0989c..90bef11164 100644 --- a/tensorflow/core/graph/mkl_tfconversion_pass_test.cc +++ b/tensorflow/core/graph/mkl_tfconversion_pass_test.cc @@ -149,7 +149,7 @@ TEST_F(MklToTfConversionPass, Positive) { " input: ['C', 'D']}"); EXPECT_EQ(DoRunMklToTfConversionPass(), "A(Input);B(Input);C(_MklConv2D);D(Input);E(Sub);M(_MklInput);" - "_Mkl2Tf/_0(_MklToTf);N(_MklInput)|A->C;B->C:2;C->Mkl2Tf/_0;" + "Mkl2Tf/_0(_MklToTf);N(_MklInput)|A->C;B->C:2;C->Mkl2Tf/_0;" "C:1->Mkl2Tf/_0:1;D->E:1;M->C:1;Mkl2Tf/_0->E;N->C:3"); } else { CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS); @@ -172,7 +172,7 @@ TEST_F(MklToTfConversionPass, Positive) { " input: ['C', 'D']}"); EXPECT_EQ(DoRunMklToTfConversionPass(), "A(Input);B(Input);C(_MklConv2D);D(Input);E(Sub);M(_MklInput);" - "_Mkl2Tf/_0(_MklToTf);N(_MklInput)|A->C;B->C:1;C->Mkl2Tf/_0;" + "Mkl2Tf/_0(_MklToTf);N(_MklInput)|A->C;B->C:1;C->Mkl2Tf/_0;" "C:1->Mkl2Tf/_0:1;D->E:1;M->C:2;Mkl2Tf/_0->E;N->C:3"); } } diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 042c839faa..75921a87b5 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -4946,6 +4946,14 @@ tf_mkl_kernel_library( ) tf_mkl_kernel_library( + name = "mkl_identity_op", + prefix = "mkl_identity_op", + deps = ARRAY_DEPS + [ + "//third_party/mkl:intel_binary_blob", + ], +) + +tf_mkl_kernel_library( name = "mkl_lrn_op", prefix = "mkl_lrn_op", deps = NN_DEPS + [ diff --git a/tensorflow/core/kernels/mkl_avgpooling_op.cc b/tensorflow/core/kernels/mkl_avgpooling_op.cc index 8bd1724e32..d90baee069 100644 --- a/tensorflow/core/kernels/mkl_avgpooling_op.cc +++ b/tensorflow/core/kernels/mkl_avgpooling_op.cc @@ -343,11 +343,10 @@ class MklAvgPoolingGradOp : public OpKernel { if (!outbackprop_in_mkl_format) { // For avgpooling, tensor_in_shape should have 1 dimension, and 4 // elements. - OP_REQUIRES( - context, - tensor_in_shape.dims() == 1 && tensor_in_shape.NumElements() == 4, - errors::InvalidArgument("original input shape must be " - "1-dimensional and 4 elements")); + OP_REQUIRES(context, tensor_in_shape.dims() == 1 && + tensor_in_shape.NumElements() == 4, + errors::InvalidArgument("original input shape must be " + "1-dimensional and 4 elements")); // For avgpooling, out_backprop should have 4 dimensions. OP_REQUIRES(context, out_backprop.dims() == 4, diff --git a/tensorflow/core/kernels/mkl_concat_op.cc b/tensorflow/core/kernels/mkl_concat_op.cc index 27930c44a6..094ab1c6c6 100644 --- a/tensorflow/core/kernels/mkl_concat_op.cc +++ b/tensorflow/core/kernels/mkl_concat_op.cc @@ -265,6 +265,7 @@ class MklConcatOp : public OpKernel { s.GetDimension() > 0 ? s.GetSizes()[concat_dim] : 1; } mkl_context.MklCreateInputLayouts(context, input_shapes); + OP_REQUIRES_OK(context, context->status()); CHECK_EQ(dnnConcatCreate_F32(&mkl_context.prim_concat, NULL, N, &mkl_context.lt_inputs[0]), @@ -316,12 +317,14 @@ class MklConcatOp : public OpKernel { mkl_context.mkl_tmp_tensors.resize(N); mkl_context.MklPrepareConcatInputs(context, input_tensors); + OP_REQUIRES_OK(context, context->status()); // Execute primitive. CHECK_EQ(dnnExecute_F32(mkl_context.prim_concat, mkl_context.concat_res), E_SUCCESS); mkl_context.MklCleanup(); + OP_REQUIRES_OK(context, context->status()); } private: diff --git a/tensorflow/core/kernels/mkl_conv_grad_bias_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_bias_ops.cc index 8a1006a8e9..d4364d31e4 100644 --- a/tensorflow/core/kernels/mkl_conv_grad_bias_ops.cc +++ b/tensorflow/core/kernels/mkl_conv_grad_bias_ops.cc @@ -38,9 +38,9 @@ limitations under the License. #include "tensorflow/core/util/use_cudnn.h" #include "tensorflow/core/util/work_sharder.h" +#include "tensorflow/core/util/mkl_util.h" #include "third_party/mkl/include/mkl_dnn.h" #include "third_party/mkl/include/mkl_dnn_types.h" -#include "tensorflow/core/util/mkl_util.h" namespace tensorflow { @@ -252,7 +252,7 @@ class MklConv2DCustomBackpropBiasOp : public OpKernel { }; #define REGISTER_CPU_KERNELS(T) \ - REGISTER_KERNEL_BUILDER(Name("_MklConv2DWithBiasBackpropBias") \ + REGISTER_KERNEL_BUILDER(Name("_MklConv2DWithBiasBackpropBias") \ .Device(DEVICE_CPU) \ .TypeConstraint<T>("T") \ .Label(mkl_op_registry::kMklOpLabel), \ diff --git a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc index 6381b527a1..dc6b88e953 100644 --- a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc +++ b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc @@ -37,9 +37,9 @@ limitations under the License. #include "tensorflow/core/util/use_cudnn.h" #include "tensorflow/core/util/work_sharder.h" +#include "tensorflow/core/util/mkl_util.h" #include "third_party/mkl/include/mkl_dnn.h" #include "third_party/mkl/include/mkl_dnn_types.h" -#include "tensorflow/core/util/mkl_util.h" namespace tensorflow { @@ -266,8 +266,11 @@ class MklConv2DCustomBackpropFilterOp : public OpKernel { int input_offsets[2]; size_t conv_strides[2]; MklShape input_shape, grad_filter_shape, out_backprop_shape; - dnnPrimitive_t prim_conv_bwdfilter, convert_bwdfilter; - dnnLayout_t lt_input, lt_grad_filter, lt_out_backprop; + dnnPrimitive_t prim_conv_bwdfilter = nullptr; + dnnPrimitive_t convert_bwdfilter = nullptr; + dnnLayout_t lt_input = nullptr; + dnnLayout_t lt_grad_filter = nullptr; + dnnLayout_t lt_out_backprop = nullptr; void* conv_res[dnnResourceNumber]; void MklCleanup() { @@ -409,7 +412,7 @@ class MklConv2DCustomBackpropFilterOp : public OpKernel { }; #define REGISTER_MKL_FILTER_KERNELS(T) \ - REGISTER_KERNEL_BUILDER(Name("_MklConv2DBackpropFilter") \ + REGISTER_KERNEL_BUILDER(Name("_MklConv2DBackpropFilter") \ .Device(DEVICE_CPU) \ .TypeConstraint<T>("T") \ .Label(mkl_op_registry::kMklOpLabel), \ diff --git a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc index 638ce4c024..c97f1dd7b7 100644 --- a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc +++ b/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc @@ -23,8 +23,6 @@ limitations under the License. #define EIGEN_USE_THREADS #include <algorithm> #include <vector> -#include "third_party/mkl/include/mkl_dnn.h" -#include "third_party/mkl/include/mkl_dnn_types.h" #include "tensorflow/core/framework/numeric_op.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" @@ -42,6 +40,8 @@ limitations under the License. #include "tensorflow/core/util/tensor_format.h" #include "tensorflow/core/util/use_cudnn.h" #include "tensorflow/core/util/work_sharder.h" +#include "third_party/mkl/include/mkl_dnn.h" +#include "third_party/mkl/include/mkl_dnn_types.h" namespace tensorflow { @@ -342,7 +342,7 @@ class MklConv2DCustomBackpropInputOp : public OpKernel { }; #define REGISTER_MKL_CPU_KERNELS(T) \ - REGISTER_KERNEL_BUILDER(Name("_MklConv2DBackpropInput") \ + REGISTER_KERNEL_BUILDER(Name("_MklConv2DBackpropInput") \ .Device(DEVICE_CPU) \ .TypeConstraint<T>("T") \ .Label(mkl_op_registry::kMklOpLabel), \ diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc index b818819b02..76b9f1798d 100644 --- a/tensorflow/core/kernels/mkl_conv_ops.cc +++ b/tensorflow/core/kernels/mkl_conv_ops.cc @@ -36,9 +36,9 @@ limitations under the License. #include "tensorflow/core/util/padding.h" #include "tensorflow/core/util/tensor_format.h" +#include "tensorflow/core/util/mkl_util.h" #include "third_party/mkl/include/mkl_dnn.h" #include "third_party/mkl/include/mkl_dnn_types.h" -#include "tensorflow/core/util/mkl_util.h" namespace tensorflow { @@ -98,19 +98,18 @@ class MklConv2DOp : public OpKernel { filter.shape().DebugString())); for (int i = 0; i < 3; i++) { - OP_REQUIRES( - context, - FastBoundsCheck(filter.dim_size(i), std::numeric_limits<int>::max()), - errors::InvalidArgument("filter too large")); + OP_REQUIRES(context, FastBoundsCheck(filter.dim_size(i), + std::numeric_limits<int>::max()), + errors::InvalidArgument("filter too large")); } const int64 input_depth = input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'C') : GetTensorDim(input, data_format_, 'C'); - OP_REQUIRES(context, input_depth == filter.dim_size(2), - errors::InvalidArgument( - "input and filter must have the same depth: ", input_depth, - " vs ", filter.dim_size(2))); + OP_REQUIRES( + context, input_depth == filter.dim_size(2), + errors::InvalidArgument("input and filter must have the same depth: ", + input_depth, " vs ", filter.dim_size(2))); // The last dimension for filter is out_depth. const int out_depth = static_cast<int>(filter.dim_size(3)); @@ -119,10 +118,9 @@ class MklConv2DOp : public OpKernel { const int64 input_rows_raw = input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'H') : GetTensorDim(input, data_format_, 'H'); - OP_REQUIRES( - context, - FastBoundsCheck(input_rows_raw, std::numeric_limits<int>::max()), - errors::InvalidArgument("Input rows too large")); + OP_REQUIRES(context, FastBoundsCheck(input_rows_raw, + std::numeric_limits<int>::max()), + errors::InvalidArgument("Input rows too large")); const int input_rows = static_cast<int>(input_rows_raw); const int filter_rows = static_cast<int>(filter.dim_size(0)); @@ -131,10 +129,9 @@ class MklConv2DOp : public OpKernel { const int64 input_cols_raw = input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'W') : GetTensorDim(input, data_format_, 'W'); - OP_REQUIRES( - context, - FastBoundsCheck(input_cols_raw, std::numeric_limits<int>::max()), - errors::InvalidArgument("Input cols too large")); + OP_REQUIRES(context, FastBoundsCheck(input_cols_raw, + std::numeric_limits<int>::max()), + errors::InvalidArgument("Input cols too large")); const int input_cols = static_cast<int>(input_cols_raw); const int filter_cols = static_cast<int>(filter.dim_size(1)); @@ -142,10 +139,9 @@ class MklConv2DOp : public OpKernel { const int64 input_batch_raw = input_in_mkl_format ? GetMklTensorDim(mkl_context.input_shape, 'N') : GetTensorDim(input, data_format_, 'N'); - OP_REQUIRES( - context, - FastBoundsCheck(input_batch_raw, std::numeric_limits<int>::max()), - errors::InvalidArgument("batch is too large")); + OP_REQUIRES(context, FastBoundsCheck(input_batch_raw, + std::numeric_limits<int>::max()), + errors::InvalidArgument("batch is too large")); const int batch = static_cast<int>(input_batch_raw); // For now we take the stride from the second and third dimensions only (we @@ -438,12 +434,12 @@ class MklConv2DOp : public OpKernel { }; #define REGISTER_MKL_CPU(T) \ - REGISTER_KERNEL_BUILDER(Name("_MklConv2D") \ + REGISTER_KERNEL_BUILDER(Name("_MklConv2D") \ .Device(DEVICE_CPU) \ .TypeConstraint<T>("T") \ .Label(mkl_op_registry::kMklOpLabel), \ MklConv2DOp<CPUDevice, T, false>); \ - REGISTER_KERNEL_BUILDER(Name("_MklConv2DWithBias") \ + REGISTER_KERNEL_BUILDER(Name("_MklConv2DWithBias") \ .Device(DEVICE_CPU) \ .TypeConstraint<T>("T") \ .Label(mkl_op_registry::kMklOpLabel), \ diff --git a/tensorflow/core/kernels/mkl_identity_op.cc b/tensorflow/core/kernels/mkl_identity_op.cc new file mode 100644 index 0000000000..e138cc2e95 --- /dev/null +++ b/tensorflow/core/kernels/mkl_identity_op.cc @@ -0,0 +1,63 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// See docs in ../ops/array_ops.cc. +#ifdef INTEL_MKL + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/logging.h" + +#include "tensorflow/core/util/mkl_util.h" +#include "third_party/mkl/include/mkl_dnn.h" +#include "third_party/mkl/include/mkl_dnn_types.h" + +namespace tensorflow { +typedef Eigen::ThreadPoolDevice CPUDevice; +template <typename Device, typename T> +class MklIdentityOp : public OpKernel { + public: + explicit MklIdentityOp(OpKernelConstruction* context) : OpKernel(context) {} + + void Compute(OpKernelContext* context) override { + MklShape mkl_shape_input; + GetMklShape(context, 0, &mkl_shape_input); + bool input_in_mkl_format = mkl_shape_input.IsMklTensor(); + + if (input_in_mkl_format) { + ForwarMklTensorInToOut(context, 0, 0); + } else { + FowardTfTensorInToOut(context, 0, 0); + } + } + + bool IsExpensive() override { return false; } +}; + +#define REGISTER_MKL_CPU(T) \ + REGISTER_KERNEL_BUILDER(Name("_MklIdentity") \ + .Device(DEVICE_CPU) \ + .TypeConstraint<T>("T") \ + .Label(mkl_op_registry::kMklOpLabel), \ + MklIdentityOp<CPUDevice, T>); \ + +TF_CALL_float(REGISTER_MKL_CPU); +#undef REGISTER_MKL_CPU +} // namespace tensorflow +#endif // INTEL_MKL diff --git a/tensorflow/core/kernels/mkl_lrn_op.cc b/tensorflow/core/kernels/mkl_lrn_op.cc index edca8e2553..ac432e13ce 100644 --- a/tensorflow/core/kernels/mkl_lrn_op.cc +++ b/tensorflow/core/kernels/mkl_lrn_op.cc @@ -104,6 +104,15 @@ class MklLRNOp : public OpKernel { return; } + // TODO(inteltf) MKL will support depth radius not equal to 2 in the future + if (depth_radius_ != 2) { + Tensor converted_tensor = + ConvertMklToTF<T>(context, input, mkl_context.input_shape); + mkl_context.MklDefaultToEigen(context, depth_radius_, bias_, alpha_, + beta_, converted_tensor); + return; + } + if (input_in_mkl_format) { // MKL supports normalization over channel dimension only if (mkl_context.input_shape.tf_dim_idx(mkl_context.in_dims - 1) == @@ -112,8 +121,10 @@ class MklLRNOp : public OpKernel { static_cast<dnnLayout_t>(mkl_context.input_shape.GetCurLayout()); workspace_enabled_ = true; } else { + Tensor converted_tensor = + ConvertMklToTF<T>(context, input, mkl_context.input_shape); mkl_context.MklDefaultToEigen(context, depth_radius_, bias_, alpha_, - beta_, input); + beta_, converted_tensor); return; } } @@ -160,9 +171,7 @@ class MklLRNOp : public OpKernel { MklShape input_shape; dnnPrimitive_t lrn_fwd = nullptr; dnnPrimitive_t convert_input = nullptr; - /* dnnPrimitive_t convert_output; */ dnnLayout_t lt_input = nullptr; - /* dnnLayout_t lt_output; */ dnnLayout_t lt_internal_input = nullptr; dnnLayout_t lt_internal_workspace = nullptr; dnnLayout_t lt_internal_output = nullptr; @@ -267,7 +276,7 @@ class MklLRNOp : public OpKernel { } // Fallback implementation - Taken from lrn_op.cc - // TODO(intelft) Check if we can use EigenLRNOp directly instead of making a + // TODO(inteltf) Check if we can use EigenLRNOp directly instead of making a // copy. void MklDefaultToEigen(OpKernelContext* context, int depth_radius_, float bias_, float alpha_, float beta_, @@ -378,6 +387,7 @@ class MklLRNGradOp : public OpKernel { mkl_context.MklDefaultToEigen(context); return; } + if (ingrad_in_mkl_format || inimage_in_mkl_format) { const MklShape* tmp_mkl_shape = (ingrad_in_mkl_format) ? &mkl_context.ingrad_shape @@ -459,11 +469,11 @@ class MklLRNGradOp : public OpKernel { const_cast<void*>(static_cast<const void*>(output->flat<T>().data())); Tensor mkl_tmp_input_buf_tensor, mkl_tmp_image_buf_tensor, - mkl_tmp_outimage_buf_tensor, mkl_tmp_workspace_buf_tensor; + mkl_tmp_outimage_buf_tensor; // Convert Inputs if needed - mkl_context.MklPrepareLRNGradInput( - context, &mkl_tmp_input_buf_tensor, &mkl_tmp_image_buf_tensor, - &mkl_tmp_outimage_buf_tensor, &mkl_tmp_workspace_buf_tensor); + mkl_context.MklPrepareLRNGradInput(context, &mkl_tmp_input_buf_tensor, + &mkl_tmp_image_buf_tensor, + &mkl_tmp_outimage_buf_tensor); // We do not do any conversion for output. But we simply emit it // in MKL format. @@ -489,14 +499,11 @@ class MklLRNGradOp : public OpKernel { MklShape ingrad_shape, inimage_shape, outimage_shape; dnnPrimitive_t lrn_bwd = nullptr; dnnPrimitive_t convert_input = nullptr; - /* dnnPrimitive_t convert_output; */ dnnLayout_t lt_input = nullptr; dnnLayout_t lt_output = nullptr; dnnLayout_t lt_bdw_input = nullptr; dnnLayout_t lt_workspace = nullptr; dnnLayout_t lt_internal_input = nullptr; - /* dnnLayout_t lt_internal_workspace; - dnnLayout_t lt_internal_output; */ void* res_lrn_bwd[dnnResourceNumber]; // prepare mkl input @@ -523,11 +530,13 @@ class MklLRNGradOp : public OpKernel { void MklPrepareLRNGradInput(OpKernelContext* context, Tensor* mkl_tmp_input_buf_tensor, Tensor* mkl_tmp_image_buf_tensor, - Tensor* mkl_tmp_outimage_buf_tensor, - Tensor* mkl_tmp_workspace_buf_tensor) { + Tensor* mkl_tmp_outimage_buf_tensor) { const Tensor& in_grads = MklGetInput(context, 0); const Tensor& in_image = MklGetInput(context, 1); const Tensor& out_image = MklGetInput(context, 2); + const Tensor& workspace = MklGetInput( + context, + 3); /*Worskpsace is enabled, get the buffer to the workspace */ void* user_input = const_cast<void*>( static_cast<const void*>(in_grads.flat<T>().data())); @@ -535,6 +544,9 @@ class MklLRNGradOp : public OpKernel { static_cast<const void*>(in_image.flat<T>().data())); void* user_fwd_output = const_cast<void*>( static_cast<const void*>(out_image.flat<T>().data())); + void* workspace_buffer = const_cast<void*>( + static_cast<const void*>(workspace.flat<T>().data())); + CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(<_workspace, lrn_bwd, dnnResourceWorkspace), E_SUCCESS); @@ -609,9 +621,7 @@ class MklLRNGradOp : public OpKernel { res_lrn_bwd[dnnResourceDst] = user_fwd_output; } - // Allocate buffer for workspace. - AllocTmpBuffer(context, mkl_tmp_workspace_buf_tensor, lt_workspace, - &res_lrn_bwd[dnnResourceWorkspace]); + res_lrn_bwd[dnnResourceWorkspace] = workspace_buffer; } // Fallback implementation - Taken from lrn_op.cc @@ -619,14 +629,36 @@ class MklLRNGradOp : public OpKernel { // copy. void MklDefaultToEigen(OpKernelContext* context) { // CHECK(false); - Tensor in_grads = MklGetInput(context, 0); - Tensor in_image = MklGetInput(context, 1); - Tensor out_image = MklGetInput(context, 2); + + Tensor in_grads; + Tensor in_image; + Tensor out_image; GetMklShape(context, 0, &ingrad_shape); GetMklShape(context, 1, &inimage_shape); GetMklShape(context, 2, &outimage_shape); + if (ingrad_shape.IsMklTensor()) { + in_grads = + ConvertMklToTF<T>(context, MklGetInput(context, 0), ingrad_shape); + } else { + in_grads = MklGetInput(context, 0); + } + + if (inimage_shape.IsMklTensor()) { + in_image = + ConvertMklToTF<T>(context, MklGetInput(context, 1), inimage_shape); + } else { + in_image = MklGetInput(context, 1); + } + + if (outimage_shape.IsMklTensor()) { + out_image = + ConvertMklToTF<T>(context, MklGetInput(context, 2), outimage_shape); + } else { + out_image = MklGetInput(context, 2); + } + const int64 batch = static_cast<int64>(in_grads.dim_size(0)); const int64 rows = static_cast<int64>(in_grads.dim_size(1)); const int64 cols = static_cast<int64>(in_grads.dim_size(2)); @@ -677,7 +709,7 @@ class MklLRNGradOp : public OpKernel { Shard(worker_threads.num_threads, worker_threads.workers, nodes * batch, depth * depth, shard); } - + // release mkl resources void Mklcleanup() { bool ingrad_in_mkl_format = ingrad_shape.IsMklTensor(); diff --git a/tensorflow/core/kernels/mkl_matmul_op.cc b/tensorflow/core/kernels/mkl_matmul_op.cc index 3ba28c13ed..e43b75e250 100644 --- a/tensorflow/core/kernels/mkl_matmul_op.cc +++ b/tensorflow/core/kernels/mkl_matmul_op.cc @@ -199,15 +199,13 @@ class MklMatMulOp : public OpKernel { } }; -#define REGISTER_CPU(T) \ - REGISTER_KERNEL_BUILDER( \ - Name("MatMul").Device(DEVICE_CPU).TypeConstraint<T>("T"), \ - MklMatMulOp<CPUDevice, T, false /* cublas, ignored for CPU */>); \ - REGISTER_KERNEL_BUILDER( \ - Name("MatMul").Device(DEVICE_CPU).TypeConstraint<T>("T").Label("MKL"), \ - MklMatMulOp<CPUDevice, T, false /* cublas, ignored for CPU */>) - -// TODO:Consider template specialization when adding/removing additional types +#define REGISTER_CPU(T) \ + REGISTER_KERNEL_BUILDER( \ + Name("MatMul").Device(DEVICE_CPU).TypeConstraint<T>("T"), \ + MklMatMulOp<CPUDevice, T, false /* cublas, ignored for CPU */>); + +// TODO(inteltf) Consider template specialization when adding/removing +// additional types TF_CALL_float(REGISTER_CPU); TF_CALL_double(REGISTER_CPU); TF_CALL_complex64(REGISTER_CPU); diff --git a/tensorflow/core/kernels/mkl_maxpooling_op.cc b/tensorflow/core/kernels/mkl_maxpooling_op.cc index e27881f882..1e0ee258b0 100644 --- a/tensorflow/core/kernels/mkl_maxpooling_op.cc +++ b/tensorflow/core/kernels/mkl_maxpooling_op.cc @@ -276,11 +276,6 @@ class MklMaxPoolingGradOp : public OpKernel { mkl_context.pooling_res[dnnResourceDiffSrc] = const_cast<void*>( static_cast<const void*>(output_tensor->flat<T>().data())); - int64 output_size = output_tensor->NumElements(); - for (int64 i = 0; i < output_size; ++i) { - (static_cast<float*>(mkl_context.pooling_res[dnnResourceDiffSrc]))[i] = 0; - } - CHECK_EQ( dnnExecute_F32(mkl_context.prim_pooling_bwd, mkl_context.pooling_res), E_SUCCESS); @@ -387,19 +382,18 @@ class MklMaxPoolingGradOp : public OpKernel { if (workspace_enabled == false) { if (convert_input != nullptr) { if (input_in_mkl_format == false) { - CHECK_EQ(dnnConversionExecute_F32( - convert_input, - const_cast<void*>(static_cast<const void*>( - tensor_in.flat<T>().data())), - input_buf), - E_SUCCESS); + CHECK_EQ( + dnnConversionExecute_F32( + convert_input, const_cast<void*>(static_cast<const void*>( + tensor_in.flat<T>().data())), + input_buf), + E_SUCCESS); CHECK_EQ(dnnDelete_F32(convert_input), E_SUCCESS); convert_input = nullptr; } else { input_shape.GetConvertedFlatData( - lt_input_prim, - const_cast<void*>( - static_cast<const void*>(tensor_in.flat<T>().data())), + lt_input_prim, const_cast<void*>(static_cast<const void*>( + tensor_in.flat<T>().data())), input_buf); } pooling_resfwd[dnnResourceSrc] = input_buf; @@ -444,9 +438,8 @@ class MklMaxPoolingGradOp : public OpKernel { CHECK_EQ(dnnDelete_F32(convert_outbackprop), E_SUCCESS); } else { output_backprop_shape.GetConvertedFlatData( - lt_outbackprop_prim, - const_cast<void*>( - static_cast<const void*>(out_backprop.flat<T>().data())), + lt_outbackprop_prim, const_cast<void*>(static_cast<const void*>( + out_backprop.flat<T>().data())), outbackprop_buf); } pooling_res[dnnResourceDiffDst] = outbackprop_buf; diff --git a/tensorflow/core/kernels/mkl_pooling_ops_common.cc b/tensorflow/core/kernels/mkl_pooling_ops_common.cc index d88bd4c640..65e8852cfb 100644 --- a/tensorflow/core/kernels/mkl_pooling_ops_common.cc +++ b/tensorflow/core/kernels/mkl_pooling_ops_common.cc @@ -14,8 +14,8 @@ limitations under the License. ==============================================================================*/ #ifdef INTEL_MKL -#include "tensorflow/core/kernels/mkl_pooling_ops_common.h" #include <vector> +#include "tensorflow/core/kernels/mkl_pooling_ops_common.h" #include "tensorflow/core/common_runtime/device.h" #include "tensorflow/core/framework/common_shape_fns.h" diff --git a/tensorflow/core/kernels/mkl_relu_op.cc b/tensorflow/core/kernels/mkl_relu_op.cc index 25c8359cc5..0c66f73141 100644 --- a/tensorflow/core/kernels/mkl_relu_op.cc +++ b/tensorflow/core/kernels/mkl_relu_op.cc @@ -16,17 +16,17 @@ limitations under the License. // See docs in ../ops/nn_ops.cc. #ifdef INTEL_MKL -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/numeric_op.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/lib/core/errors.h" +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "third_party/mkl/include/mkl_dnn.h" -#include "third_party/mkl/include/mkl_dnn_types.h" #include "tensorflow/core/platform/default/logging.h" #include "tensorflow/core/util/mkl_util.h" +#include "third_party/mkl/include/mkl_dnn.h" +#include "third_party/mkl/include/mkl_dnn_types.h" namespace tensorflow { @@ -194,45 +194,29 @@ class MklReluGradOp : public OpKernel { void* user_i = static_cast<void*>(const_cast<T*>(a.flat<T>().data())); void* user_g = static_cast<void*>(const_cast<T*>(g.flat<T>().data())); - - CHECK_EQ(dnnLayoutCreateFromPrimitive_F32( - &mkl_lt_internal_grad, prim_relu_bwd, dnnResourceDiffDst), - E_SUCCESS); - - CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(&mkl_lt_internal_input, - prim_relu_bwd, dnnResourceSrc), - E_SUCCESS); - - if (!dnnLayoutCompare_F32(mkl_lt_internal_grad, lt_grad)) { - AllocTmpBuffer(context, mkl_tmp_grad_buf_tensor, mkl_lt_internal_grad, - &relu_res[dnnResourceDiffDst]); - CHECK_EQ(dnnConversionCreate_F32(&cv_user_to_reluB_grad, lt_grad, - mkl_lt_internal_grad), + dnnPrimitive_t cv_input_to_grad = NULL; + Tensor mkl_tmp_buf_tensor; + void* mkl_buffer_convert = nullptr; + + // if input and grad are not in the same layout, do a conversion between + // them. + if (!dnnLayoutCompare_F32(lt_input, lt_grad)) { + AllocTmpBuffer(context, &mkl_tmp_buf_tensor, lt_grad, + &mkl_buffer_convert); + CHECK_EQ(dnnConversionCreate_F32(&cv_input_to_grad, lt_input, lt_grad), E_SUCCESS); - CHECK_EQ(dnnConversionExecute_F32(cv_user_to_reluB_grad, user_g, - relu_res[dnnResourceDiffDst]), - E_SUCCESS); - dnnDelete_F32(cv_user_to_reluB_grad); - } else { - relu_res[dnnResourceDiffDst] = user_g; - } - if (!dnnLayoutCompare_F32(mkl_lt_internal_input, lt_input)) { - AllocTmpBuffer(context, mkl_tmp_input_buf_tensor, mkl_lt_internal_input, - &relu_res[dnnResourceSrc]); - CHECK_EQ(dnnConversionCreate_F32(&cv_user_to_reluB_input, lt_input, - mkl_lt_internal_input), - E_SUCCESS); - CHECK_EQ(dnnConversionExecute_F32(cv_user_to_reluB_input, user_i, - relu_res[dnnResourceSrc]), + CHECK_EQ(dnnConversionExecute_F32(cv_input_to_grad, user_i, + mkl_buffer_convert), E_SUCCESS); - dnnDelete_F32(cv_user_to_reluB_input); + relu_res[dnnResourceSrc] = mkl_buffer_convert; + dnnDelete_F32(cv_input_to_grad); } else { relu_res[dnnResourceSrc] = user_i; } - dnnLayoutDelete_F32(mkl_lt_internal_input); - dnnLayoutDelete_F32(mkl_lt_internal_grad); + relu_res[dnnResourceDiffDst] = user_g; + } void MklCreateInputLayouts(OpKernelContext* context) { @@ -331,7 +315,7 @@ void MklReluGradOp<Device, T>::Compute(OpKernelContext* context) { mkl_context.MklCreateInputLayouts(context); float negative_slope = 0.0; CHECK_EQ(dnnReLUCreateBackward_F32(&mkl_context.prim_relu_bwd, NULL, - mkl_context.lt_grad, mkl_context.lt_input, + mkl_context.lt_grad, mkl_context.lt_grad, negative_slope), E_SUCCESS); Tensor mkl_tmp_grad_buf_tensor, mkl_tmp_input_buf_tensor; @@ -380,12 +364,12 @@ void MklReluGradOp<Device, T>::Compute(OpKernelContext* context) { /* Register DNN kernels for supported operations and supported types - right now * it is only Relu and f32*/ #define REGISTER_RELU_MKL_SUPPORTED_KERNELS_TYPES(type) \ - REGISTER_KERNEL_BUILDER(Name("_MklRelu") \ + REGISTER_KERNEL_BUILDER(Name("_MklRelu") \ .Device(DEVICE_CPU) \ .TypeConstraint<type>("T") \ .Label(mkl_op_registry::kMklOpLabel), \ MklReluOp<CPUDevice, type>); \ - REGISTER_KERNEL_BUILDER(Name("_MklReluGrad") \ + REGISTER_KERNEL_BUILDER(Name("_MklReluGrad") \ .Device(DEVICE_CPU) \ .TypeConstraint<type>("T") \ .Label(mkl_op_registry::kMklOpLabel), \ diff --git a/tensorflow/core/kernels/mkl_reshape_op.cc b/tensorflow/core/kernels/mkl_reshape_op.cc index 753a8b52b4..593aa3a2fd 100644 --- a/tensorflow/core/kernels/mkl_reshape_op.cc +++ b/tensorflow/core/kernels/mkl_reshape_op.cc @@ -129,7 +129,7 @@ class MklReshapeOp : public OpKernel { return; } } else { - CopyTFTensorInToOut(context, 0, 0, shape); + CopyTfTensorInToOutWithShape(context, 0, 0, shape); } } }; diff --git a/tensorflow/core/kernels/mkl_tfconv_op.cc b/tensorflow/core/kernels/mkl_tfconv_op.cc index c31ef5c255..588d6874dd 100644 --- a/tensorflow/core/kernels/mkl_tfconv_op.cc +++ b/tensorflow/core/kernels/mkl_tfconv_op.cc @@ -106,7 +106,7 @@ class MklToTfOp : public OpKernel { /////////////////////////////////////////////////////////// #define REGISTER_CPU(T) \ - REGISTER_KERNEL_BUILDER(Name("MklToTf") \ + REGISTER_KERNEL_BUILDER(Name("_MklToTf") \ .Device(DEVICE_CPU) \ .TypeConstraint<T>("T") \ .Label(mkl_op_registry::kMklOpLabel), \ diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc index cf68680bb3..a1d09bd503 100644 --- a/tensorflow/core/ops/array_ops.cc +++ b/tensorflow/core/ops/array_ops.cc @@ -1542,6 +1542,23 @@ REGISTER_OP("Identity") Return a tensor with the same shape and contents as the input tensor or value. )Doc"); +#ifdef INTEL_MKL +REGISTER_OP("_MklIdentity") + .Input("input: T") + .Input("mkl_input: uint8") + .Output("output: T") + .Output("mkl_output: uint8") + .Attr("T: type") + .SetShapeFn([](shape_inference::InferenceContext* c) { + c->set_output(0, c->input(0)); + c->set_output_handle_dtype(0, c->input_handle_dtype(0)); + c->set_output_handle_shape(0, c->input_handle_shape(0)); + return Status::OK(); + }) + .Doc(R"Doc( Mkl implementation of IdentityOp +)Doc"); +#endif + // -------------------------------------------------------------------------- REGISTER_OP("RefIdentity") .Input("input: Ref(T)") diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h index 897b174eff..6a37256ea9 100644 --- a/tensorflow/core/util/mkl_util.h +++ b/tensorflow/core/util/mkl_util.h @@ -542,8 +542,8 @@ inline int64 GetMklTensorDim(const MklShape& mkl_shape, char dimension) { return mkl_shape.dim_size(index); } -inline void CopyMklTensorInToOut(OpKernelContext* context, int idx_in, - int idx_out) { +inline void CopyMklTensorInToOut(OpKernelContext* context, + int idx_in, int idx_out) { int num_inputs = context->num_inputs(); int num_outputs = context->num_outputs(); int idx_data_in = GetTensorDataIndex(idx_in, num_inputs); @@ -563,8 +563,9 @@ inline void CopyMklTensorInToOut(OpKernelContext* context, int idx_in, context->set_output(idx_meta_out, meta_output); } -inline void CopyTFTensorInToOut(OpKernelContext* context, int idx_in, - int idx_out, const TensorShape& shape) { +inline void CopyTfTensorInToOutWithShape(OpKernelContext* context, + int idx_in, int idx_out, + const TensorShape& shape) { int num_inputs = context->num_inputs(); int num_outputs = context->num_outputs(); int idx_data_in = GetTensorDataIndex(idx_in, num_inputs); @@ -580,6 +581,41 @@ inline void CopyTFTensorInToOut(OpKernelContext* context, int idx_in, context->set_output(idx_data_out, output); } +inline void FowardTfTensorInToOut(OpKernelContext* context, + int idx_in, int idx_out) { + int num_inputs = context->num_inputs(); + int num_outputs = context->num_outputs(); + int idx_data_in = GetTensorDataIndex(idx_in, num_inputs); + int idx_data_out = GetTensorDataIndex(idx_out, num_outputs); + + MklShape mkl_shape_output; + mkl_shape_output.SetMklTensor(false); + AllocateOutputSetMklShape(context, idx_out, mkl_shape_output); + if (IsRefType(context->input_dtype(idx_data_in))) { + context->forward_ref_input_to_ref_output(idx_data_in, idx_data_out); + } else { + context->set_output(idx_data_out, context->input(idx_data_in)); + } +} + +inline void ForwarMklTensorInToOut(OpKernelContext* context, + int idx_in, int idx_out) { + int num_inputs = context->num_inputs(); + int num_outputs = context->num_outputs(); + int idx_data_in = GetTensorDataIndex(idx_in, num_inputs); + int idx_meta_in = GetTensorMetaDataIndex(idx_in, num_inputs); + int idx_data_out = GetTensorDataIndex(idx_out, num_outputs); + int idx_meta_out = GetTensorMetaDataIndex(idx_out, num_outputs); + + if (IsRefType(context->input_dtype(idx_data_in))) { + context->forward_ref_input_to_ref_output(idx_data_in, idx_data_out); + context->forward_ref_input_to_ref_output(idx_meta_in, idx_meta_out); + } else { + context->set_output(idx_data_out, context->input(idx_data_in)); + context->set_output(idx_meta_out, context->input(idx_meta_in)); + } +} + namespace mkl_op_registry { static const char* kMklOpLabel = "MklOp"; static const char* kMklOpLabelPattern = "label='MklOp'"; diff --git a/tensorflow/docs_src/community/style_guide.md b/tensorflow/docs_src/community/style_guide.md index 767e33c3d0..f90a6cf938 100644 --- a/tensorflow/docs_src/community/style_guide.md +++ b/tensorflow/docs_src/community/style_guide.md @@ -162,9 +162,9 @@ operation. - `reuse`: `bool` indicator if the variable should be reused if it's present in the scope. -* Layers that behave differently during training should have: - - `is_training`: `bool` to indicate if a training graph is been built. - +* Layers that behave differently during training should take: + - `is_training`: `bool` indicator to conditionally choose different + computation paths (e.g. using `tf.cond`) during execution. Example: diff --git a/tensorflow/docs_src/extend/adding_an_op.md b/tensorflow/docs_src/extend/adding_an_op.md index 99933b75b4..3586794418 100644 --- a/tensorflow/docs_src/extend/adding_an_op.md +++ b/tensorflow/docs_src/extend/adding_an_op.md @@ -42,7 +42,7 @@ To incorporate your custom op you'll need to: Python @{tf.test.compute_gradient_error$gradient checker}. See [`relu_op_test.py`](https://www.tensorflow.org/code/tensorflow/python/kernel_tests/relu_op_test.py) as - an example that does tests the forward functions of Relu-like operators and + an example that tests the forward functions of Relu-like operators and their gradients. PREREQUISITES: @@ -345,7 +345,7 @@ building the `.so` file. > the older ABI. If you compile your op library with `gcc>=5`, add > `-D_GLIBCXX_USE_CXX11_ABI=0` to the command line to make the library > compatible with the older abi. -> Furthermore if you are using TensorFlow package created from source remember to add `-cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0"` +> Furthermore if you are using TensorFlow package created from source remember to add `--cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0"` > as bazel command to compile the Python package. ### Compile the op using bazel (TensorFlow source installation) @@ -382,7 +382,7 @@ TensorFlow Python API provides the load the dynamic library and register the op with the TensorFlow framework. `load_op_library` returns a Python module that contains the Python wrappers for the op and the kernel. Thus, once you have built the op, you can -do the following to run it from Python : +do the following to run it from Python: ```python import tensorflow as tf diff --git a/tensorflow/docs_src/get_started/monitors.md b/tensorflow/docs_src/get_started/monitors.md index cb4ef70eeb..d9c605b013 100644 --- a/tensorflow/docs_src/get_started/monitors.md +++ b/tensorflow/docs_src/get_started/monitors.md @@ -65,7 +65,7 @@ if __name__ == "__main__": Copy the above code into a file, and download the corresponding [training](http://download.tensorflow.org/data/iris_training.csv) and -@{tf.test} data sets to the same +[test](http://download.tensorflow.org/data/iris_test.csv) data sets to the same directory. In the following sections, you'll progressively make updates to the above code diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md index 2a907b6e82..72d0c7b1ff 100644 --- a/tensorflow/docs_src/install/install_java.md +++ b/tensorflow/docs_src/install/install_java.md @@ -106,7 +106,7 @@ As an example, these steps will create a Maven project that uses TensorFlow: The preceding command should output <tt>Hello from <i>version</i></tt>. If it -does, you've succesfully set up TensorFlow for Java and are ready to use it in +does, you've successfully set up TensorFlow for Java and are ready to use it in Maven projects. If not, check [Stack Overflow](http://stackoverflow.com/questions/tagged/tensorflow) for possible solutions. You can skip reading the rest of this document. diff --git a/tensorflow/docs_src/performance/performance_models.md b/tensorflow/docs_src/performance/performance_models.md index d415c29aa1..9ca1e77ff5 100644 --- a/tensorflow/docs_src/performance/performance_models.md +++ b/tensorflow/docs_src/performance/performance_models.md @@ -140,7 +140,7 @@ performance and increase the flexiblity of models. Most TensorFlow operations used by a CNN support both NHWC and NCHW data format. On GPU, NCHW is faster. But on CPU, NHWC is sometimes faster. -Building a model to support both date formats keeps the model flexible and +Building a model to support both data formats keeps the model flexible and capable of operating optimally regardless of platform. Most TensorFlow operations used by a CNN support both NHWC and NCHW data format. The benchmark script was written to support both NCHW and NHWC. NCHW should always be used diff --git a/tensorflow/examples/label_image/main.cc b/tensorflow/examples/label_image/main.cc index ad032f93fd..3351109f45 100644 --- a/tensorflow/examples/label_image/main.cc +++ b/tensorflow/examples/label_image/main.cc @@ -308,11 +308,11 @@ int main(int argc, char* argv[]) { } // This is for automated testing to make sure we get the expected result with - // the default settings. We know that label 866 (military uniform) should be + // the default settings. We know that label 653 (military uniform) should be // the top label for the Admiral Hopper image. if (self_test) { bool expected_matches; - Status check_status = CheckTopLabel(outputs, 866, &expected_matches); + Status check_status = CheckTopLabel(outputs, 653, &expected_matches); if (!check_status.ok()) { LOG(ERROR) << "Running check failed: " << check_status; return -1; diff --git a/tensorflow/examples/tutorials/estimators/abalone.py b/tensorflow/examples/tutorials/estimators/abalone.py index 932ce8a8b2..3c0ea2e409 100644 --- a/tensorflow/examples/tutorials/estimators/abalone.py +++ b/tensorflow/examples/tutorials/estimators/abalone.py @@ -134,12 +134,22 @@ def main(unused_argv): # Instantiate Estimator nn = tf.contrib.learn.Estimator(model_fn=model_fn, params=model_params) - + + def get_train_inputs(): + x = tf.constant(training_set.data) + y = tf.constant(training_set.target) + return x, y + # Fit - nn.fit(x=training_set.data, y=training_set.target, steps=5000) + nn.fit(input_fn=get_train_inputs, steps=5000) # Score accuracy - ev = nn.evaluate(x=test_set.data, y=test_set.target, steps=1) + def get_test_inputs(): + x = tf.constant(test_set.data) + y = tf.constant(test_set.target) + return x, y + + ev = nn.evaluate(input_fn=get_test_inputs, steps=1) print("Loss: %s" % ev["loss"]) print("Root Mean Squared Error: %s" % ev["rmse"]) diff --git a/tensorflow/java/BUILD b/tensorflow/java/BUILD index f2904ad5a6..a8910248c1 100644 --- a/tensorflow/java/BUILD +++ b/tensorflow/java/BUILD @@ -55,6 +55,18 @@ java_test( ) java_test( + name = "OperationTest", + size = "small", + srcs = ["src/test/java/org/tensorflow/OperationTest.java"], + test_class = "org.tensorflow.OperationTest", + deps = [ + ":tensorflow", + ":testutil", + "@junit", + ], +) + +java_test( name = "SavedModelBundleTest", size = "small", srcs = ["src/test/java/org/tensorflow/SavedModelBundleTest.java"], diff --git a/tensorflow/java/src/main/java/org/tensorflow/Operation.java b/tensorflow/java/src/main/java/org/tensorflow/Operation.java index 48db554e07..43dbaf125c 100644 --- a/tensorflow/java/src/main/java/org/tensorflow/Operation.java +++ b/tensorflow/java/src/main/java/org/tensorflow/Operation.java @@ -70,6 +70,28 @@ public final class Operation { } } + /** + * Returns the size of the list of Tensors produced by this operation. + * + * <p>An Operation has multiple named outputs, each of which produces either + * a single tensor or a list of tensors. This method returns the size of + * the list of tensors for a specific named output of the operation. + * + * @param name identifier of the list of tensors (of which there may + * be many) produced by this operation. + * @returns the size of the list of Tensors produced by this named output. + * @throws IllegalArgumentException if this operation has no output + * with the provided name. + */ + public int outputListLength(final String name) { + Graph.Reference r = graph.ref(); + try { + return outputListLength(unsafeNativeHandle, name); + } finally { + r.close(); + } + } + /** Returns a symbolic handle to one of the tensors produced by this operation. */ public Output output(int idx) { return new Output(this, idx); @@ -108,6 +130,8 @@ public final class Operation { private static native int numOutputs(long handle); + private static native int outputListLength(long handle, String name); + private static native long[] shape(long graphHandle, long opHandle, int output); private static native int dtype(long graphHandle, long opHandle, int output); diff --git a/tensorflow/java/src/main/native/operation_jni.cc b/tensorflow/java/src/main/native/operation_jni.cc index 32e59bc0ae..b3d5fc4ec3 100644 --- a/tensorflow/java/src/main/native/operation_jni.cc +++ b/tensorflow/java/src/main/native/operation_jni.cc @@ -66,6 +66,24 @@ JNIEXPORT jint JNICALL Java_org_tensorflow_Operation_numOutputs(JNIEnv* env, return TF_OperationNumOutputs(op); } +JNIEXPORT jint JNICALL Java_org_tensorflow_Operation_outputListLength(JNIEnv* env, + jclass clazz, + jlong handle, + jstring name) { + TF_Operation* op = requireHandle(env, handle); + if (op == nullptr) return 0; + + TF_Status* status = TF_NewStatus(); + + const char* cname = env->GetStringUTFChars(name, nullptr); + int result = TF_OperationOutputListLength(op, cname, status); + env->ReleaseStringUTFChars(name, cname); + + throwExceptionIfNotOK(env, status); + TF_DeleteStatus(status); + return result; +} + JNIEXPORT jlongArray JNICALL Java_org_tensorflow_Operation_shape( JNIEnv* env, jclass clazz, jlong graph_handle, jlong op_handle, jint output_index) { diff --git a/tensorflow/java/src/main/native/operation_jni.h b/tensorflow/java/src/main/native/operation_jni.h index 6292a48069..b5d156f7c2 100644 --- a/tensorflow/java/src/main/native/operation_jni.h +++ b/tensorflow/java/src/main/native/operation_jni.h @@ -48,6 +48,16 @@ JNIEXPORT jint JNICALL Java_org_tensorflow_Operation_numOutputs(JNIEnv *, /* * Class: org_tensorflow_Operation + * Method: outputListLength + * Signature: (JLjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_tensorflow_Operation_outputListLength(JNIEnv *, + jclass, + jlong, + jstring); + +/* + * Class: org_tensorflow_Operation * Method: shape * Signature: (JJI)[J */ diff --git a/tensorflow/java/src/test/java/org/tensorflow/OperationTest.java b/tensorflow/java/src/test/java/org/tensorflow/OperationTest.java new file mode 100644 index 0000000000..53bd511b5b --- /dev/null +++ b/tensorflow/java/src/test/java/org/tensorflow/OperationTest.java @@ -0,0 +1,66 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.List; + +/** Unit tests for {@link org.tensorflow.Operation}. */ +@RunWith(JUnit4.class) +public class OperationTest { + + @Test + public void outputListLengthFailsOnInvalidName() { + try (Graph g = new Graph()) { + Operation op = g.opBuilder("Add", "Add") + .addInput(TestUtil.constant(g, "x", 1)) + .addInput(TestUtil.constant(g, "y", 2)) + .build(); + assertEquals(1, op.outputListLength("z")); + + try { + op.outputListLength("unknown"); + fail("Did not catch bad name"); + } catch (IllegalArgumentException iae) { + // expected + } + } + } + + @Test + public void outputListLength() { + assertEquals(1, split(new int[]{0, 1}, 1)); + assertEquals(2, split(new int[]{0, 1}, 2)); + assertEquals(3, split(new int[]{0, 1, 2}, 3)); + } + + private int split(int[] values, int num_split) { + try (Graph g = new Graph()) { + return g.opBuilder("Split", "Split") + .addInput(TestUtil.constant(g, "split_dim", 0)) + .addInput(TestUtil.constant(g, "values", values)) + .setAttr("num_split", num_split) + .build() + .outputListLength("output"); + } + } +} diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py index 7981defe78..700e95c0b9 100644 --- a/tensorflow/python/client/session.py +++ b/tensorflow/python/client/session.py @@ -594,6 +594,9 @@ class BaseSession(SessionInterface): try: status = tf_session.TF_NewStatus() tf_session.TF_DeleteDeprecatedSession(self._session, status) + except AttributeError: + # 'NoneType' object has no attribute 'TF_NewStatus' + pass finally: if status is not None: tf_session.TF_DeleteStatus(status) diff --git a/tensorflow/python/debug/cli/debugger_cli_common.py b/tensorflow/python/debug/cli/debugger_cli_common.py index 64a22e6be4..889fc6a8f6 100644 --- a/tensorflow/python/debug/cli/debugger_cli_common.py +++ b/tensorflow/python/debug/cli/debugger_cli_common.py @@ -648,7 +648,7 @@ class CommandHandlerRegistry(object): 3) the handler is found for the prefix, but it fails to return a RichTextLines or raise any exception. CommandLineExit: - If the command handler raises this type of exception, tihs method will + If the command handler raises this type of exception, this method will simply pass it along. """ if not prefix: diff --git a/tensorflow/python/debug/cli/profile_analyzer_cli.py b/tensorflow/python/debug/cli/profile_analyzer_cli.py index 520d472c5c..3837717767 100644 --- a/tensorflow/python/debug/cli/profile_analyzer_cli.py +++ b/tensorflow/python/debug/cli/profile_analyzer_cli.py @@ -56,7 +56,7 @@ class ProfileDatum(object): @property def exec_time(self): - """Measures compute function exection time plus pre- and post-processing.""" + """Measures compute function execution time plus pre- and post-processing.""" return self.node_exec_stats.all_end_rel_micros diff --git a/tensorflow/python/debug/lib/debug_data.py b/tensorflow/python/debug/lib/debug_data.py index ce4bc82e0a..de1e1ce017 100644 --- a/tensorflow/python/debug/lib/debug_data.py +++ b/tensorflow/python/debug/lib/debug_data.py @@ -975,7 +975,7 @@ class DebugDumpDir(object): slot = datum.output_slot # In some cases (e.g., system clocks with insufficient precision), # the upstream and downstream tensors may have identical timestamps, the - # following check examines this possibilty and avoids raising an error if + # following check examines this possibility and avoids raising an error if # that is the case. if not self._satisfied_at_timestamp( pending_inputs[node], datum.timestamp, start_i=i + 1): diff --git a/tensorflow/python/debug/wrappers/hooks.py b/tensorflow/python/debug/wrappers/hooks.py index c27448e283..f6194f5fad 100644 --- a/tensorflow/python/debug/wrappers/hooks.py +++ b/tensorflow/python/debug/wrappers/hooks.py @@ -66,7 +66,7 @@ class LocalCLIDebugHook(session_run_hook.SessionRunHook, """Add a tensor filter. See doc of `LocalCLIDebugWrapperSession.add_tensor_filter()` for details. - Override default behavior to accomodate the possibility of this method being + Override default behavior to accommodate the possibility of this method being called prior to the initialization of the underlying `LocalCLIDebugWrapperSession` object. diff --git a/tensorflow/python/framework/importer.py b/tensorflow/python/framework/importer.py index fcddd9546d..ed579224d3 100644 --- a/tensorflow/python/framework/importer.py +++ b/tensorflow/python/framework/importer.py @@ -275,6 +275,9 @@ def import_graph_def(graph_def, input_map=None, return_elements=None, # 1. Add operations without their inputs. for node in graph_def.node: + # Check to see if this op's name matches a previously seen op + if node.name in name_to_op: + raise ValueError('Duplicate name \'%s\' in GraphDef.' % node.name) # Set any default attr values that aren't present. if node.op not in op_dict: raise ValueError('No op named %s in defined operations.' % node.op) diff --git a/tensorflow/python/framework/importer_test.py b/tensorflow/python/framework/importer_test.py index c4ccc3d189..2b2398f833 100644 --- a/tensorflow/python/framework/importer_test.py +++ b/tensorflow/python/framework/importer_test.py @@ -685,6 +685,17 @@ class ImportGraphDefTest(test.TestCase): self.assertEqual("return_elements must be a list of strings.", str(e.exception)) + def testDuplicateOperationNames(self): + with ops.Graph().as_default(): + with self.assertRaises(ValueError) as e: + importer.import_graph_def( + self._MakeGraphDef(""" + node { name: 'A' op: 'Oi' } + node { name: 'B' op: 'Oi' } + node { name: 'A' op: 'Oi' } + """)) + self.assertEqual("Duplicate name 'A' in GraphDef.", str(e.exception)) + def testWithExtensionAndAttr(self): with ops.Graph().as_default() as g: c = constant_op.constant(5.0, dtype=dtypes.float32, name="c") diff --git a/tensorflow/python/framework/subscribe.py b/tensorflow/python/framework/subscribe.py index 91c6e33f22..2654bca31c 100644 --- a/tensorflow/python/framework/subscribe.py +++ b/tensorflow/python/framework/subscribe.py @@ -276,7 +276,7 @@ def subscribe(tensors, side_effects): Subscribed tensors, which are identity copies of the passed in tensors in the same passed in structure, but the graph has been modified such that these are downstream of the control dependencies for - the side effect graphs. Use these functionally equivelant tensors + the side effect graphs. Use these functionally equivalent tensors instead of the passed in tensors for further construction or running. """ if not hasattr(side_effects, '__iter__'): diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py index c3169e23a5..ac551a6e1a 100644 --- a/tensorflow/python/framework/test_util.py +++ b/tensorflow/python/framework/test_util.py @@ -250,7 +250,7 @@ class TensorFlowTestCase(googletest.TestCase): """Returns a unique temporary directory for the test to use. If you call this method multiple times during in a test, it will return the - same folder. However, accross different runs the directories will be + same folder. However, across different runs the directories will be different. This will ensure that across different runs tests will not be able to pollute each others environment. If you need multiple unique directories within a single test, you should diff --git a/tensorflow/python/kernel_tests/linalg_ops_test.py b/tensorflow/python/kernel_tests/linalg_ops_test.py index 153d4ab662..2d31ac85b0 100644 --- a/tensorflow/python/kernel_tests/linalg_ops_test.py +++ b/tensorflow/python/kernel_tests/linalg_ops_test.py @@ -28,7 +28,7 @@ from tensorflow.python.platform import test def _random_pd_matrix(n, rng): - """Random postive definite matrix.""" + """Random positive definite matrix.""" temp = rng.randn(n, n) return temp.dot(temp.T) diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py index f92ea9b05f..b3d28f010f 100644 --- a/tensorflow/python/layers/normalization.py +++ b/tensorflow/python/layers/normalization.py @@ -322,7 +322,7 @@ class BatchNormalization(base.Layer): def _broadcast(v): if needs_broadcasting and v is not None: - # In this case we must explictly broadcast all parameters. + # In this case we must explicitly broadcast all parameters. return array_ops.reshape(v, broadcast_shape) return v diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index f3bcb6ce77..f1c4d922e0 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -1690,21 +1690,21 @@ def meshgrid(*args, **kwargs): results in ```prettyprint - X = [[1, 1, 1], - [2, 2, 2], - [3, 3, 3]] - Y = [[4, 5, 6], - [4, 5, 6], - [4, 5, 6]] + X = [[1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + Y = [[4, 4, 4], + [5, 5, 5], + [6, 6, 6]] ``` Args: - *args: `Tensor`s with rank 1 - indexing: Either 'xy' or 'ij' (optional, default: 'xy') + *args: `Tensor`s with rank 1. + indexing: Either 'xy' or 'ij' (optional, default: 'xy'). name: A name for the operation (optional). Returns: - outputs: A list of N `Tensor`s with rank N + outputs: A list of N `Tensor`s with rank N. """ indexing = kwargs.pop("indexing", "xy") diff --git a/tensorflow/python/ops/ctc_ops.py b/tensorflow/python/ops/ctc_ops.py index ee5b2952f4..69edaa2c40 100644 --- a/tensorflow/python/ops/ctc_ops.py +++ b/tensorflow/python/ops/ctc_ops.py @@ -35,11 +35,9 @@ def ctc_loss(labels, inputs, sequence_length, This op implements the CTC loss as presented in the article: - A. Graves, S. Fernandez, F. Gomez, J. Schmidhuber. + [A. Graves, S. Fernandez, F. Gomez, J. Schmidhuber. Connectionist Temporal Classification: Labelling Unsegmented Sequence Data - with Recurrent Neural Networks. ICML 2006, Pittsburgh, USA, pp. 369-376. - - http://www.cs.toronto.edu/~graves/icml_2006.pdf + with Recurrent Neural Networks. ICML 2006, Pittsburgh, USA, pp. 369-376.](http://www.cs.toronto.edu/~graves/icml_2006.pdf) Input requirements: diff --git a/tensorflow/python/ops/gradients_impl.py b/tensorflow/python/ops/gradients_impl.py index 2adf8f05d8..bd8a5c86ac 100644 --- a/tensorflow/python/ops/gradients_impl.py +++ b/tensorflow/python/ops/gradients_impl.py @@ -273,28 +273,6 @@ def _VerifyGeneratedGradients(grads, op): if len(grads) != len(op.inputs): raise ValueError("Num gradients %d generated for op %s do not match num " "inputs %d" % (len(grads), op.node_def, len(op.inputs))) - for i in xrange(len(grads)): - grad = grads[i] - inp = op.inputs[i] - if grad is None: - continue - if grad.dtype.is_floating: - if not inp.dtype.is_floating: - raise TypeError("Gradient type %s generated for real-valued op %s " - "with type %s must be real" % - (dtypes.as_dtype(grad.dtype).name, op.node_def, - dtypes.as_dtype(inp.dtype).name)) - elif grad.dtype.is_complex: - if not inp.dtype.is_complex: - raise TypeError("Gradient type %s generated for complex-valued op %s" - " with type %s must be complex" % - (dtypes.as_dtype(grad.dtype).name, op.node_def, - dtypes.as_dtype(inp.dtype).name)) - else: - raise TypeError("Gradient type %s generated for op %s " - "with type %s must be either real or complex" % - (dtypes.as_dtype(grad.dtype).name, op.node_def, - dtypes.as_dtype(inp.dtype).name)) def _StopOps(from_ops, pending_count): diff --git a/tensorflow/python/ops/init_ops.py b/tensorflow/python/ops/init_ops.py index 42b4f952bb..1e2f999995 100644 --- a/tensorflow/python/ops/init_ops.py +++ b/tensorflow/python/ops/init_ops.py @@ -41,6 +41,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops +from tensorflow.python.ops import math_ops class Initializer(object): diff --git a/tensorflow/python/ops/logging_ops.py b/tensorflow/python/ops/logging_ops.py index dbb13e0c8f..08e3f83a0b 100644 --- a/tensorflow/python/ops/logging_ops.py +++ b/tensorflow/python/ops/logging_ops.py @@ -86,7 +86,7 @@ def histogram_summary(tag, values, collections=None, name=None): This ops is deprecated. Please switch to tf.summary.histogram. For an explanation of why this op was deprecated, and information on how to - migrate, look ['here'](https://www.tensorflow.org/code/tensorflow/contrib/deprecated/__init__.py) + migrate, look ['here'](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/deprecated/__init__.py) The generated [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto) @@ -190,7 +190,7 @@ def audio_summary(tag, This op is deprecated. Please switch to tf.summary.audio. For an explanation of why this op was deprecated, and information on how to - migrate, look ['here'](https://www.tensorflow.org/code/tensorflow/contrib/deprecated/__init__.py) + migrate, look ['here'](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/deprecated/__init__.py) The summary has up to `max_outputs` summary values containing audio. The audio is built from `tensor` which must be 3-D with shape `[batch_size, @@ -326,7 +326,7 @@ def scalar_summary(tags, values, collections=None, name=None): This ops is deprecated. Please switch to tf.summary.scalar. For an explanation of why this op was deprecated, and information on how to - migrate, look ['here'](https://www.tensorflow.org/code/tensorflow/contrib/deprecated/__init__.py) + migrate, look ['here'](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/deprecated/__init__.py) The input `tags` and `values` must have the same shape. The generated summary has a summary value for each tag-value pair in `tags` and `values`. diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py index b762205cf7..158016ff37 100644 --- a/tensorflow/python/ops/math_ops.py +++ b/tensorflow/python/ops/math_ops.py @@ -2331,7 +2331,7 @@ def tensordot(a, b, axes, name=None): using `array_ops.transpose` and `array_ops.reshape`. The method takes a tensor and performs the correct transpose and reshape operation for a given set of indices. It returns the reshaped tensor as well as a list of indices - necesary to reshape the tensor again after matrix multiplication. + necessary to reshape the tensor again after matrix multiplication. Args: a: `Tensor`. diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py index 28ed3af9d7..0d35f50894 100644 --- a/tensorflow/python/ops/metrics_impl.py +++ b/tensorflow/python/ops/metrics_impl.py @@ -1735,7 +1735,7 @@ def _streaming_sparse_true_positive_at_k(labels, A tuple of `Variable` and update `Operation`. Raises: - ValueError: If `weights` is not `None` and has an incomptable shape. + ValueError: If `weights` is not `None` and has an incompatible shape. """ with ops.name_scope( name, _at_k_name('true_positive', k, class_id=class_id), @@ -1831,7 +1831,7 @@ def _streaming_sparse_false_negative_at_k(labels, A tuple of `Variable` and update `Operation`. Raises: - ValueError: If `weights` is not `None` and has an incomptable shape. + ValueError: If `weights` is not `None` and has an incompatible shape. """ with ops.name_scope( name, _at_k_name('false_negative', k, class_id=class_id), @@ -2653,7 +2653,7 @@ def _streaming_sparse_false_positive_at_k(labels, A tuple of `Variable` and update `Operation`. Raises: - ValueError: If `weights` is not `None` and has an incomptable shape. + ValueError: If `weights` is not `None` and has an incompatible shape. """ with ops.name_scope( name, _at_k_name('false_positive', k, class_id=class_id), diff --git a/tensorflow/python/ops/parsing_ops.py b/tensorflow/python/ops/parsing_ops.py index 796ea20eb7..c2f9961731 100644 --- a/tensorflow/python/ops/parsing_ops.py +++ b/tensorflow/python/ops/parsing_ops.py @@ -58,7 +58,7 @@ class SparseFeature( ["index_key", "value_key", "dtype", "size", "already_sorted"])): """Configuration for parsing a sparse input feature from an `Example`. - Note, preferrably use `VarLenFeature` (possibly in combination with a + Note, preferably use `VarLenFeature` (possibly in combination with a `SequenceExample`) in order to parse out `SparseTensor`s instead of `SparseFeature` due to its simplicity. diff --git a/tensorflow/python/tools/saved_model_cli.py b/tensorflow/python/tools/saved_model_cli.py index e19b187681..e1be305505 100644 --- a/tensorflow/python/tools/saved_model_cli.py +++ b/tensorflow/python/tools/saved_model_cli.py @@ -489,7 +489,7 @@ def show(args): else: # If no tag is specified, display all tag_set, if no signaure_def key is # specified, display all SignatureDef keys, else show input output tensor - # infomation corresponding to the given SignatureDef key + # information corresponding to the given SignatureDef key if args.tag_set is None: _show_tag_sets(args.dir) else: @@ -562,7 +562,7 @@ def create_parser(): parser_show.add_argument( '--all', action='store_true', - help='if set, will output all infomation in given SavedModel') + help='if set, will output all information in given SavedModel') parser_show.add_argument( '--tag_set', type=str, diff --git a/tensorflow/python/training/coordinator.py b/tensorflow/python/training/coordinator.py index fea2f8240e..d234df71c1 100644 --- a/tensorflow/python/training/coordinator.py +++ b/tensorflow/python/training/coordinator.py @@ -366,7 +366,7 @@ class Coordinator(object): # If any thread is still alive, wait for the grace period to expire. # By the time this check is executed, threads may still be shutting down, # so we add a sleep of increasing duration to give them a chance to shut - # down without loosing too many cycles. + # down without losing too many cycles. # The sleep duration is limited to the remaining grace duration. stop_wait_secs = 0.001 while any(t.is_alive() for t in threads) and stop_grace_period_secs >= 0.0: diff --git a/tensorflow/python/util/deprecation.py b/tensorflow/python/util/deprecation.py index 73fc3e2408..1e1599afb4 100644 --- a/tensorflow/python/util/deprecation.py +++ b/tensorflow/python/util/deprecation.py @@ -182,7 +182,7 @@ def deprecated_args(date, instructions, *deprecated_arg_names_or_tuples): return d def _get_deprecated_positional_arguments(names_to_ok_vals, arg_spec): - """Builds a dictionary from deprecated arguments to thier spec. + """Builds a dictionary from deprecated arguments to their spec. Returned dict is keyed by argument name. Each value is a DeprecatedArgSpec with the following fields: diff --git a/tensorflow/stream_executor/lib/statusor.h b/tensorflow/stream_executor/lib/statusor.h index 06278d5152..bb423e390a 100644 --- a/tensorflow/stream_executor/lib/statusor.h +++ b/tensorflow/stream_executor/lib/statusor.h @@ -135,7 +135,7 @@ class StatusOr { // operators, to support move-only types and avoid unnecessary copying. StatusOr(T&& value); // NOLINT - // Move conversion operator to avoid unecessary copy. + // Move conversion operator to avoid unnecessary copy. // T must be assignable from U. // Not marked with explicit so the implicit conversion can happen. template <typename U> diff --git a/tensorflow/tensorboard/backend/event_processing/event_multiplexer_test.py b/tensorflow/tensorboard/backend/event_processing/event_multiplexer_test.py index ded1856d7e..a97f39e87f 100644 --- a/tensorflow/tensorboard/backend/event_processing/event_multiplexer_test.py +++ b/tensorflow/tensorboard/backend/event_processing/event_multiplexer_test.py @@ -124,16 +124,19 @@ class EventMultiplexerTest(test_util.TensorFlowTestCase): self.stubs.CleanUp() def testEmptyLoader(self): + """Tests empty EventMultiplexer creation.""" x = event_multiplexer.EventMultiplexer() self.assertEqual(x.Runs(), {}) def testRunNamesRespected(self): + """Tests two EventAccumulators inserted/accessed in EventMultiplexer.""" x = event_multiplexer.EventMultiplexer({'run1': 'path1', 'run2': 'path2'}) self.assertItemsEqual(sorted(x.Runs().keys()), ['run1', 'run2']) self.assertEqual(x._GetAccumulator('run1')._path, 'path1') self.assertEqual(x._GetAccumulator('run2')._path, 'path2') def testReload(self): + """EventAccumulators should Reload after EventMultiplexer call it.""" x = event_multiplexer.EventMultiplexer({'run1': 'path1', 'run2': 'path2'}) self.assertFalse(x._GetAccumulator('run1').reload_called) self.assertFalse(x._GetAccumulator('run2').reload_called) @@ -142,6 +145,7 @@ class EventMultiplexerTest(test_util.TensorFlowTestCase): self.assertTrue(x._GetAccumulator('run2').reload_called) def testScalars(self): + """Tests Scalars function returns suitable values.""" x = event_multiplexer.EventMultiplexer({'run1': 'path1', 'run2': 'path2'}) run1_actual = x.Scalars('run1', 'sv1') @@ -150,6 +154,7 @@ class EventMultiplexerTest(test_util.TensorFlowTestCase): self.assertEqual(run1_expected, run1_actual) def testHealthPills(self): + """Tests HealthPills() returns events associated with run1/Add.""" self.stubs.Set(event_accumulator, 'EventAccumulator', functools.partial( _GetFakeAccumulator, @@ -172,11 +177,13 @@ class EventMultiplexerTest(test_util.TensorFlowTestCase): self.assertItemsEqual(['Add'], x.GetOpsWithHealthPills('run1')) def testExceptions(self): + """KeyError should be raised when accessing non-existing keys.""" x = event_multiplexer.EventMultiplexer({'run1': 'path1', 'run2': 'path2'}) with self.assertRaises(KeyError): x.Scalars('sv1', 'xxx') def testInitialization(self): + """Tests EventMultiplexer is created properly with its params.""" x = event_multiplexer.EventMultiplexer() self.assertEqual(x.Runs(), {}) x = event_multiplexer.EventMultiplexer({'run1': 'path1', 'run2': 'path2'}) @@ -185,6 +192,14 @@ class EventMultiplexerTest(test_util.TensorFlowTestCase): self.assertEqual(x._GetAccumulator('run2')._path, 'path2') def testAddRunsFromDirectory(self): + """Tests AddRunsFromDirectory function. + + Tests the following scenarios: + - When the directory does not exist. + - When the directory is empty. + - When the directory has empty subdirectory. + - Contains proper EventAccumulators after adding events. + """ x = event_multiplexer.EventMultiplexer() tmpdir = self.get_temp_dir() join = os.path.join diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index ff4222032d..348745f8d2 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -1185,7 +1185,7 @@ def tf_version_info_genrule(): ], outs=["util/version_info.cc"], cmd= - "$(PYTHON_BIN_PATH) $(location //tensorflow/tools/git:gen_git_source.py) --generate $(SRCS) \"$@\"", + "$(location //tensorflow/tools/git:gen_git_source.py) --generate $(SRCS) \"$@\"", local=1, tools=[clean_dep("//tensorflow/tools/git:gen_git_source.py")],) diff --git a/tensorflow/tools/ci_build/install/install_auditwheel.sh b/tensorflow/tools/ci_build/install/install_auditwheel.sh index 2538a393d3..e6f6124d56 100755 --- a/tensorflow/tools/ci_build/install/install_auditwheel.sh +++ b/tensorflow/tools/ci_build/install/install_auditwheel.sh @@ -16,7 +16,7 @@ set -e -sudo pip3 install auditwheel +sudo pip3 install auditwheel==1.5.0 set +e patchelf_location=$(which patchelf) diff --git a/tensorflow/tools/ci_build/install/install_deb_packages.sh b/tensorflow/tools/ci_build/install/install_deb_packages.sh index 6b160bbe03..da1f2199d0 100755 --- a/tensorflow/tools/ci_build/install/install_deb_packages.sh +++ b/tensorflow/tools/ci_build/install/install_deb_packages.sh @@ -46,6 +46,7 @@ apt-get install -y --no-install-recommends \ git \ libcurl4-openssl-dev \ libtool \ + mlocate \ openjdk-8-jdk \ openjdk-8-jre-headless \ pkg-config \ @@ -63,6 +64,9 @@ apt-get install -y --no-install-recommends \ zip \ zlib1g-dev +# populate the database +updatedb + if [[ "$1" != "--without_cmake" ]]; then apt-get install -y --no-install-recommends \ cmake diff --git a/tensorflow/tools/ci_build/linux/cmake/run.sh b/tensorflow/tools/ci_build/linux/cmake/run.sh index d9bf4f01b5..d9bf4f01b5 100755..100644 --- a/tensorflow/tools/ci_build/linux/cmake/run.sh +++ b/tensorflow/tools/ci_build/linux/cmake/run.sh diff --git a/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh b/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh index 762c531725..d90a1b905d 100755 --- a/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh +++ b/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh @@ -28,6 +28,7 @@ export TF_NEED_GCP=0 export TF_NEED_HDFS=0 export TF_NEED_CUDA=0 export TF_NEED_OPENCL=0 +export TF_NEED_MKL=0 export COMPUTECPP_PATH="/usr/local" export PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" diff --git a/tensorflow/tools/ci_build/osx/libtensorflow_gpu.sh b/tensorflow/tools/ci_build/osx/libtensorflow_gpu.sh index 1da5e8c2bf..79973647c1 100755 --- a/tensorflow/tools/ci_build/osx/libtensorflow_gpu.sh +++ b/tensorflow/tools/ci_build/osx/libtensorflow_gpu.sh @@ -29,6 +29,7 @@ export PYTHON_BIN_PATH="/usr/bin/python" export TF_NEED_GCP=0 export TF_NEED_HDFS=0 export TF_NEED_OPENCL=0 +export TF_NEED_MKL=0 export COMPUTECPP_PATH="/usr/local" export PATH="/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" diff --git a/tensorflow/tools/ci_build/windows/bazel/common_env.sh b/tensorflow/tools/ci_build/windows/bazel/common_env.sh index b993747521..e4e3861710 100644 --- a/tensorflow/tools/ci_build/windows/bazel/common_env.sh +++ b/tensorflow/tools/ci_build/windows/bazel/common_env.sh @@ -34,6 +34,7 @@ export BAZEL_SH="C:/tools/msys64/usr/bin/bash" # Set Python path for ./configure export PYTHON_BIN_PATH="C:/Program Files/Anaconda3/python" +export PYTHON_LIB_PATH="C:/Program Files/Anaconda3/lib/site-packages" # Set Python path for cc_configure.bzl export BAZEL_PYTHON="C:/Program Files/Anaconda3/python" diff --git a/tensorflow/tools/docs/py_guide_parser.py b/tensorflow/tools/docs/py_guide_parser.py index 3ca6d11b84..245643cb32 100644 --- a/tensorflow/tools/docs/py_guide_parser.py +++ b/tensorflow/tools/docs/py_guide_parser.py @@ -34,7 +34,7 @@ def md_files_in_dir(py_guide_src_dir): class PyGuideParser(object): """Simple parsing of a guide .md file. - Decendents can override the process_*() functions (called by process()) + Descendants can override the process_*() functions (called by process()) to either record infromation from the guide, or call replace_line() to affect the return value of process(). """ diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 84c0e89b91..b8cd04323c 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -8,6 +8,19 @@ load("@io_bazel_rules_closure//closure:defs.bzl", "web_library_external") load("//third_party/py:python_configure.bzl", "python_configure") +def _is_windows(repository_ctx): + """Returns true if the host operating system is windows.""" + return repository_ctx.os.name.lower().find("windows") != -1 + + +def _get_env_var(repository_ctx, name): + """Find an environment variable.""" + if name in repository_ctx.os.environ: + return repository_ctx.os.environ[name] + else: + return None + + # Parse the bazel version string from `native.bazel_version`. def _parse_bazel_version(bazel_version): # Remove commit from version. @@ -74,7 +87,7 @@ temp_workaround_http_archive = repository_rule( # Executes specified command with arguments and calls 'fail' if it exited with # non-zero code def _execute_and_check_ret_code(repo_ctx, cmd_and_args): - result = repo_ctx.execute(cmd_and_args) + result = repo_ctx.execute(cmd_and_args, timeout=10) if result.return_code != 0: fail(("Non-zero return code({1}) when executing '{0}':\n" + "Stdout: {2}\n" + "Stderr: {3}").format(" ".join(cmd_and_args), result.return_code, @@ -84,9 +97,15 @@ def _execute_and_check_ret_code(repo_ctx, cmd_and_args): # Apply a patch_file to the repository root directory # Runs 'patch -p1' def _apply_patch(repo_ctx, patch_file): - _execute_and_check_ret_code(repo_ctx, [ + cmd = [ "patch", "-p1", "-d", repo_ctx.path("."), "-i", repo_ctx.path(patch_file) - ]) + ] + if _is_windows(repo_ctx): + bazel_sh = _get_env_var(repo_ctx, "BAZEL_SH") + if not bazel_sh: + fail("BAZEL_SH environment variable is not set") + cmd = [bazel_sh, "-c", " ".join(cmd)] + _execute_and_check_ret_code(repo_ctx, cmd) # Download the repository and apply a patch to its root diff --git a/third_party/eigen3/unsupported/Eigen/CXX11/Tensor b/third_party/eigen3/unsupported/Eigen/CXX11/Tensor index 00d2e7c0c7..861a87b68b 100644 --- a/third_party/eigen3/unsupported/Eigen/CXX11/Tensor +++ b/third_party/eigen3/unsupported/Eigen/CXX11/Tensor @@ -1,9 +1,11 @@ -#ifdef _WIN32 -#define sleep(seconds) Sleep(1000*seconds) -#endif // _WIN32 #include "unsupported/Eigen/CXX11/Tensor" #ifdef _WIN32 +#ifndef SLEEP_FUNC_HEADER_GUARD +#define SLEEP_FUNC_HEADER_GUARD +inline void sleep(unsigned int seconds) { Sleep(1000*seconds); } +#endif + // On Windows, Eigen will include Windows.h, which defines various // macros that conflict with TensorFlow symbols. Undefine them here to // prevent clashes. diff --git a/third_party/grpc.BUILD b/third_party/grpc.BUILD index 1d1e2222de..1699f6a854 100644 --- a/third_party/grpc.BUILD +++ b/third_party/grpc.BUILD @@ -176,6 +176,7 @@ cc_library( ".", "include", ], + linkopts = ["-lpthread"], deps = [ ], ) @@ -1782,6 +1783,7 @@ cc_library( ".", "include", ], + linkopts = ["-lpthread"], deps = [ ":gpr", ":grpc_unsecure", diff --git a/third_party/jemalloc.BUILD b/third_party/jemalloc.BUILD index 8ed13c51a5..3a9a9a80f2 100644 --- a/third_party/jemalloc.BUILD +++ b/third_party/jemalloc.BUILD @@ -94,6 +94,9 @@ cc_library( "@%ws%//tensorflow:linux_ppc64le": [ "-lpthread", ], + "@%ws%//tensorflow:linux_x86_64": [ + "-lpthread", + ], "//conditions:default": [ ], }), diff --git a/third_party/llvm/llvm.BUILD b/third_party/llvm/llvm.BUILD index 15aa53962d..005ae21ee8 100644 --- a/third_party/llvm/llvm.BUILD +++ b/third_party/llvm/llvm.BUILD @@ -1714,6 +1714,10 @@ cc_library( "include/llvm/Support/DataTypes.h", "include/llvm/ExecutionEngine/ObjectMemoryBuffer.h", ], + linkopts = [ + "-lpthread", + "-ldl", + ], deps = [ ":config", ":demangle", diff --git a/third_party/mkl/BUILD b/third_party/mkl/BUILD index 7e95ebd355..8c86766eff 100644 --- a/third_party/mkl/BUILD +++ b/third_party/mkl/BUILD @@ -16,6 +16,7 @@ load( cc_library( name = "intel_binary_blob", srcs = if_mkl([ + "libdl.so.2", "libmklml_intel.so", "libiomp5.so", ]), diff --git a/third_party/py/python_configure.bzl b/third_party/py/python_configure.bzl index 928bd333c6..c453645db5 100644 --- a/third_party/py/python_configure.bzl +++ b/third_party/py/python_configure.bzl @@ -58,21 +58,53 @@ def _is_windows(repository_ctx): return False +def _execute(repository_ctx, cmdline, error_msg=None, error_details=None, + empty_stdout_fine=False): + """Executes an arbitrary shell command. + + Args: + repository_ctx: the repository_ctx object + cmdline: list of strings, the command to execute + error_msg: string, a summary of the error if the command fails + error_details: string, details about the error or steps to fix it + empty_stdout_fine: bool, if True, an empty stdout result is fine, otherwise + it's an error + Return: + the result of repository_ctx.execute(cmdline) + """ + result = repository_ctx.execute(cmdline) + if result.stderr or not (empty_stdout_fine or result.stdout): + _python_configure_fail( + "\n".join([ + error_msg.strip() if error_msg else "Repository command failed", + result.stderr.strip(), + error_details if error_details else ""])) + return result + + def _symlink_genrule_for_dir(repository_ctx, src_dir, dest_dir, genrule_name): """returns a genrule to symlink all files in a directory.""" # Get the list of files under this directory find_result = None if _is_windows(repository_ctx): - find_result = repository_ctx.execute([ - "dir", src_dir, "/b", "/s", "/a-d", - ]) + find_result = _execute( + repository_ctx, + ["cmd.exe", "/c", "dir", src_dir.replace("/", "\\"), "/b", "/s", + "/a-d"], + empty_stdout_fine=True) + # src_files will be used to compute BUILD rules, where path must use + # forward slashes. + src_files = find_result.stdout.replace("\\", "/").splitlines() + # Create a list with the src_dir stripped to use for outputs. + fwdslashes_src_dir = src_dir.replace("\\", "/") + dest_files = [e.replace(fwdslashes_src_dir, "") for e in src_files] else: - find_result = repository_ctx.execute([ - "find", src_dir, "-follow", "-type", "f", - ]) - # Create a list with the src_dir stripped to use for outputs. - dest_files = find_result.stdout.replace(src_dir, '').splitlines() - src_files = find_result.stdout.splitlines() + find_result = _execute( + repository_ctx, ["find", src_dir, "-follow", "-type", "f"], + empty_stdout_fine=True) + # Create a list with the src_dir stripped to use for outputs. + dest_files = find_result.stdout.replace(src_dir, '').splitlines() + src_files = find_result.stdout.splitlines() command = [] command_windows = [] outs = [] @@ -136,26 +168,27 @@ def _check_python_bin(repository_ctx, python_bin): def _get_python_include(repository_ctx, python_bin): """Gets the python include path.""" - result = repository_ctx.execute([python_bin, "-c", - 'from __future__ import print_function;' + - 'from distutils import sysconfig;' + - 'print(sysconfig.get_python_inc())']) - if result == "": - _python_configure_fail( - "Problem getting python include path. Is distutils installed?") + result = _execute(repository_ctx, + [python_bin, "-c", + 'from __future__ import print_function;' + + 'from distutils import sysconfig;' + + 'print(sysconfig.get_python_inc())'], + error_msg="Problem getting python include path.", + error_details=("Is the Python binary path set up right? " + + "(See ./configure or BAZEL_BIN_PATH.) " + + "Is distutils installed?")) return result.stdout.splitlines()[0] def _get_numpy_include(repository_ctx, python_bin): """Gets the numpy include path.""" - result = repository_ctx.execute([python_bin, "-c", - 'from __future__ import print_function;' + - 'import numpy;' + - ' print(numpy.get_include());']) - if result == "": - _python_configure_fail( - "Problem getting numpy include path. Is numpy installed?") - return result.stdout.splitlines()[0] + return _execute(repository_ctx, + [python_bin, "-c", + 'from __future__ import print_function;' + + 'import numpy;' + + ' print(numpy.get_include());'], + error_msg="Problem getting numpy include path.", + error_details="Is numpy installed?").stdout.splitlines()[0] def _create_local_python_repository(repository_ctx): diff --git a/tools/tf_env_collect.sh b/tools/tf_env_collect.sh index 71b17f4b7b..abeebeadea 100644..100755 --- a/tools/tf_env_collect.sh +++ b/tools/tf_env_collect.sh @@ -42,7 +42,7 @@ fi echo >> $OUTPUT_FILE echo '== compiler =====================================================' >> $OUTPUT_FILE -c++ --version &>> $OUTPUT_FILE +c++ --version 2>&1 >> $OUTPUT_FILE echo >> $OUTPUT_FILE echo '== uname -a =====================================================' >> $OUTPUT_FILE @@ -50,7 +50,7 @@ uname -a >> $OUTPUT_FILE echo >> $OUTPUT_FILE echo '== check pips ===================================================' >> $OUTPUT_FILE -pip list 2>&1 | grep "proto\|numpy\|tensorflow" &>> $OUTPUT_FILE +pip list 2>&1 | grep "proto\|numpy\|tensorflow" >> $OUTPUT_FILE echo >> $OUTPUT_FILE @@ -67,7 +67,7 @@ print("tf.COMPILER_VERSION = %s" % tf.GIT_VERSION) with tf.Session() as sess: print("Sanity check: %r" % sess.run(tf.constant([1,2,3])[:1])) EOF -python /tmp/check_tf.py &>> ${OUTPUT_FILE} +python /tmp/check_tf.py 2>&1 >> ${OUTPUT_FILE} DEBUG_LD=libs python -c "import tensorflow" 2>>${OUTPUT_FILE} > /tmp/loadedlibs grep libcudnn.so /tmp/loadedlibs >> $OUTPUT_FILE @@ -88,7 +88,7 @@ fi echo >> $OUTPUT_FILE >> $OUTPUT_FILE echo '== nvidia-smi ===================================================' >> $OUTPUT_FILE -nvidia-smi &>> $OUTPUT_FILE +nvidia-smi 2>&1 >> $OUTPUT_FILE echo >> $OUTPUT_FILE |